From 30ccc681f23e5f1a97fe4d1a029bc6e2c271c397 Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Wed, 20 May 2026 16:22:36 +0930 Subject: [PATCH 01/53] chore: Remove modules and related tests - Deleted `ladybug.py`, `markdown.py`, `ontology.py`, `production.py`, `production_queries.py`, `question_query_registry.py`, `verification.py`, and their corresponding test files. - Cleaned up unused imports and code related to the removed modules. - Ensured that the codebase is streamlined by eliminating obsolete functionalities. --- README.md | 25 -- src/codebase_graph/__init__.py | 22 -- src/codebase_graph/__main__.py | 4 - src/codebase_graph/cli.py | 5 - src/codebase_graph/code_map.py | 224 ------------ src/codebase_graph/context_builder.py | 8 - src/codebase_graph/document_layers.py | 56 --- src/codebase_graph/graph_context.py | 58 ---- src/codebase_graph/graph_core.py | 323 ------------------ src/codebase_graph/ladybug.py | 77 ----- src/codebase_graph/markdown.py | 32 -- src/codebase_graph/ontology.py | 79 ----- src/codebase_graph/production.py | 245 ------------- src/codebase_graph/production_queries.py | 40 --- src/codebase_graph/question_query_registry.py | 124 ------- src/codebase_graph/verification.py | 32 -- tests/conftest.py | 9 - tests/test_cli.py | 20 -- tests/test_graph_core.py | 51 --- tests/test_question_query_registry.py | 61 ---- 20 files changed, 1495 deletions(-) delete mode 100644 src/codebase_graph/__init__.py delete mode 100644 src/codebase_graph/__main__.py delete mode 100644 src/codebase_graph/cli.py delete mode 100644 src/codebase_graph/code_map.py delete mode 100644 src/codebase_graph/context_builder.py delete mode 100644 src/codebase_graph/document_layers.py delete mode 100644 src/codebase_graph/graph_context.py delete mode 100644 src/codebase_graph/graph_core.py delete mode 100644 src/codebase_graph/ladybug.py delete mode 100644 src/codebase_graph/markdown.py delete mode 100644 src/codebase_graph/ontology.py delete mode 100644 src/codebase_graph/production.py delete mode 100644 src/codebase_graph/production_queries.py delete mode 100644 src/codebase_graph/question_query_registry.py delete mode 100644 src/codebase_graph/verification.py delete mode 100644 tests/conftest.py delete mode 100644 tests/test_cli.py delete mode 100644 tests/test_graph_core.py delete mode 100644 tests/test_question_query_registry.py diff --git a/README.md b/README.md index a39482f..e4051bc 100644 --- a/README.md +++ b/README.md @@ -7,28 +7,3 @@ ```bash python -m pip install -e .[dev] ``` - -## Basic usage - -```python -from codebase_graph import CodebaseGraph - -graph = CodebaseGraph(source_root=".", state_dir=".codebase_graph/graph") -graph.materialize() -graph.search("FastAPI routes") -graph.context("SomeClass") -graph.cypher("MATCH (n:PythonClass) RETURN n.label LIMIT 5") -``` - -## CLI - -```bash -codebase-graph status --source-root . -codebase-graph materialize --source-root . -codebase-graph schema -codebase-graph search "query" -codebase-graph context "SomeClass" -codebase-graph cypher "MATCH (n:PythonClass) RETURN n.label LIMIT 5" -``` - -The base package is intentionally small and importable without optional graph database or parquet bindings. Optional storage backends can be installed through extras as they mature. diff --git a/src/codebase_graph/__init__.py b/src/codebase_graph/__init__.py deleted file mode 100644 index 280256d..0000000 --- a/src/codebase_graph/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -from .graph_core import CodebaseGraph, GraphCoreStatus -from .ladybug import ( - DEFAULT_EMBEDDING_DIMENSIONS, - HashingEmbeddingProvider, - LadybugGraphExport, - LadybugGraphExporter, - LadybugGraphStore, - LadybugUnavailableError, -) -from .ontology import ONTOLOGY_NAME - -__all__ = [ - "CodebaseGraph", - "DEFAULT_EMBEDDING_DIMENSIONS", - "GraphCoreStatus", - "HashingEmbeddingProvider", - "LadybugGraphExport", - "LadybugGraphExporter", - "LadybugGraphStore", - "LadybugUnavailableError", - "ONTOLOGY_NAME", -] diff --git a/src/codebase_graph/__main__.py b/src/codebase_graph/__main__.py deleted file mode 100644 index bfdcd0c..0000000 --- a/src/codebase_graph/__main__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .cli import main - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/src/codebase_graph/cli.py b/src/codebase_graph/cli.py deleted file mode 100644 index ce80e54..0000000 --- a/src/codebase_graph/cli.py +++ /dev/null @@ -1,5 +0,0 @@ -from __future__ import annotations - -from .graph_core import main - -__all__ = ["main"] diff --git a/src/codebase_graph/code_map.py b/src/codebase_graph/code_map.py deleted file mode 100644 index f98d3b4..0000000 --- a/src/codebase_graph/code_map.py +++ /dev/null @@ -1,224 +0,0 @@ -from __future__ import annotations - -import ast -import hashlib -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - -CODE_EXTENSIONS = {".py"} -MAX_INDEXED_FILE_BYTES = 1_000_000 -EXCLUDED_FILENAMES = {".DS_Store"} -EXCLUDED_PARTS = { - ".git", - ".hg", - ".mypy_cache", - ".pytest_cache", - ".ruff_cache", - ".tox", - ".venv", - ".codebase_graph", - "__pycache__", - "build", - "dist", - "htmlcov", - "node_modules", - "site-packages", -} - -@dataclass(slots=True) -class CodeSymbol: - id: str - label: str - kind: str - path: str - module_name: str - qualified_name: str - line_start: int | None = None - line_end: int | None = None - decorators: list[str] = field(default_factory=list) - bases: list[str] = field(default_factory=list) - summary: str = "" - -@dataclass(slots=True) -class CodeFile: - id: str - path: str - module_name: str - language: str - line_count: int - summary: str = "" - imports: list[str] = field(default_factory=list) - calls: list[str] = field(default_factory=list) - symbols: list[CodeSymbol] = field(default_factory=list) - -@dataclass(slots=True) -class CodebaseMap: - files: list[CodeFile] - - def as_dict(self) -> dict[str, Any]: - return {"files": [_file_as_dict(file) for file in self.files]} - -class CodebaseGraphBuilder: - def __init__(self, root: str | Path) -> None: - self.root = Path(root) - - def build(self) -> CodebaseMap: - files = [_parse_python_file(path, self.root) for path in _iter_python_files(self.root)] - return CodebaseMap(files=files) - -def is_excluded_codebase_path_parts(parts: tuple[str, ...]) -> bool: - return any(part in EXCLUDED_PARTS for part in parts) - -def _iter_indexable_files(root: Path, suffixes: set[str], *, case_insensitive_suffixes: bool = False) -> list[Path]: - if not root.exists(): - return [] - paths: list[Path] = [] - for path in root.rglob("*"): - if _is_indexable_file(path, root, suffixes, case_insensitive_suffixes=case_insensitive_suffixes): - paths.append(path) - return sorted(paths) - -def _is_indexable_file( - path: Path, - root: Path, - suffixes: set[str], - *, - case_insensitive_suffixes: bool = False, -) -> bool: - if not path.is_file() or path.name in EXCLUDED_FILENAMES: - return False - suffix = path.suffix.lower() if case_insensitive_suffixes else path.suffix - if suffix not in suffixes: - return False - try: - rel_parts = path.relative_to(root).parts - except ValueError: - return False - if is_excluded_codebase_path_parts(rel_parts): - return False - try: - return path.stat().st_size <= MAX_INDEXED_FILE_BYTES - except OSError: - return False - -def _iter_python_files(root: Path) -> list[Path]: - return _iter_indexable_files(root, CODE_EXTENSIONS) - -def _parse_python_file(path: Path, root: Path) -> CodeFile: - rel_path = path.relative_to(root).as_posix() - text = path.read_text(encoding="utf-8", errors="replace") - module_name = _module_name(rel_path) - try: - tree = ast.parse(text) - except SyntaxError: - return CodeFile(_id("file", rel_path), rel_path, module_name, "python", len(text.splitlines()), "Syntax error") - - imports: list[str] = [] - calls: list[str] = [] - symbols: list[CodeSymbol] = [] - for node in ast.walk(tree): - if isinstance(node, ast.Import): - imports.extend(alias.name for alias in node.names) - elif isinstance(node, ast.ImportFrom): - module = "." * node.level + (node.module or "") - imports.append(module) - elif isinstance(node, ast.Call): - calls.append(_call_name(node.func)) - - for node in tree.body: - if isinstance(node, ast.ClassDef): - class_qn = f"{module_name}.{node.name}" if module_name else node.name - symbols.append( - CodeSymbol( - id=_id("symbol", class_qn), - label=node.name, - kind="python_class", - path=rel_path, - module_name=module_name, - qualified_name=class_qn, - line_start=getattr(node, "lineno", None), - line_end=getattr(node, "end_lineno", None), - decorators=[_call_name(item) for item in node.decorator_list], - bases=[_call_name(item) for item in node.bases], - summary=ast.get_docstring(node) or "", - ) - ) - for child in node.body: - if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)): - method_qn = f"{class_qn}.{child.name}" - symbols.append( - CodeSymbol( - id=_id("symbol", method_qn), - label=child.name, - kind="python_method", - path=rel_path, - module_name=module_name, - qualified_name=method_qn, - line_start=getattr(child, "lineno", None), - line_end=getattr(child, "end_lineno", None), - decorators=[_call_name(item) for item in child.decorator_list], - summary=ast.get_docstring(child) or "", - ) - ) - elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): - function_qn = f"{module_name}.{node.name}" if module_name else node.name - symbols.append( - CodeSymbol( - id=_id("symbol", function_qn), - label=node.name, - kind="python_function", - path=rel_path, - module_name=module_name, - qualified_name=function_qn, - line_start=getattr(node, "lineno", None), - line_end=getattr(node, "end_lineno", None), - decorators=[_call_name(item) for item in node.decorator_list], - summary=ast.get_docstring(node) or "", - ) - ) - - return CodeFile( - id=_id("file", rel_path), - path=rel_path, - module_name=module_name, - language="python", - line_count=len(text.splitlines()), - summary=ast.get_docstring(tree) or "", - imports=sorted(set(imports)), - calls=sorted({call for call in calls if call}), - symbols=symbols, - ) - -def _module_name(rel_path: str) -> str: - without_suffix = rel_path[:-3] if rel_path.endswith(".py") else rel_path - parts = without_suffix.split("/") - if parts[-1] == "__init__": - parts = parts[:-1] - return ".".join(part for part in parts if part) - -def _call_name(node: ast.AST) -> str: - if isinstance(node, ast.Name): - return node.id - if isinstance(node, ast.Attribute): - base = _call_name(node.value) - return f"{base}.{node.attr}" if base else node.attr - if isinstance(node, ast.Constant): - return repr(node.value) - return "" - -def _id(prefix: str, value: str) -> str: - return f"{prefix}:{hashlib.sha1(value.encode('utf-8')).hexdigest()[:20]}" - -def _file_as_dict(file: CodeFile) -> dict[str, Any]: - return { - "id": file.id, - "path": file.path, - "module_name": file.module_name, - "language": file.language, - "line_count": file.line_count, - "summary": file.summary, - "imports": file.imports, - "calls": file.calls, - "symbols": [symbol.__dict__ for symbol in file.symbols], - } diff --git a/src/codebase_graph/context_builder.py b/src/codebase_graph/context_builder.py deleted file mode 100644 index 7a98dc4..0000000 --- a/src/codebase_graph/context_builder.py +++ /dev/null @@ -1,8 +0,0 @@ -from __future__ import annotations - -from typing import Any - -from .graph_context import build_compact_graph_context - -def assemble_context(query: str, graph: dict[str, Any], *, budget: int = 1200) -> dict[str, Any]: - return build_compact_graph_context(graph, query, budget=budget, limit=5, include_raw=False) diff --git a/src/codebase_graph/document_layers.py b/src/codebase_graph/document_layers.py deleted file mode 100644 index 0e1cb37..0000000 --- a/src/codebase_graph/document_layers.py +++ /dev/null @@ -1,56 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -import re - -@dataclass(slots=True) -class LogicalChunk: - id: str - heading: str - text: str - ordinal: int - -class LogicalChunker: - def __init__(self, max_chars: int = 1600) -> None: - self.max_chars = max_chars - - def chunk(self, text: str) -> list[LogicalChunk]: - sections = _split_sections(text) - chunks: list[LogicalChunk] = [] - for index, (heading, body) in enumerate(sections): - body = body.strip() - if not body: - continue - for part_index, part in enumerate(_split_body(body, self.max_chars)): - suffix = f"-{part_index}" if part_index else "" - chunks.append(LogicalChunk(id=f"chunk-{index}{suffix}", heading=heading, text=part, ordinal=len(chunks))) - if not chunks and text.strip(): - chunks.append(LogicalChunk(id="chunk-0", heading="Document", text=text.strip()[: self.max_chars], ordinal=0)) - return chunks - -def _split_sections(text: str) -> list[tuple[str, str]]: - matches = list(re.finditer(r"^(#{1,6})\s+(.+)$", text, flags=re.MULTILINE)) - if not matches: - return [("Document", text)] - sections: list[tuple[str, str]] = [] - for index, match in enumerate(matches): - start = match.end() - end = matches[index + 1].start() if index + 1 < len(matches) else len(text) - sections.append((match.group(2).strip(), text[start:end])) - return sections - -def _split_body(body: str, max_chars: int) -> list[str]: - if len(body) <= max_chars: - return [body] - paragraphs = [part.strip() for part in body.split("\n\n") if part.strip()] - chunks: list[str] = [] - current = "" - for paragraph in paragraphs: - if current and len(current) + len(paragraph) + 2 > max_chars: - chunks.append(current) - current = paragraph - else: - current = f"{current}\n\n{paragraph}".strip() - if current: - chunks.append(current) - return chunks or [body[:max_chars]] diff --git a/src/codebase_graph/graph_context.py b/src/codebase_graph/graph_context.py deleted file mode 100644 index c5e1eb7..0000000 --- a/src/codebase_graph/graph_context.py +++ /dev/null @@ -1,58 +0,0 @@ -from __future__ import annotations - -from typing import Any - -def build_compact_graph_context( - graph: dict[str, Any], - query: str, - *, - kind: str | None = None, - profile: str = "dependencies", - limit: int = 3, - max_depth: int = 1, - budget: int = 600, - include_raw: bool = False, -) -> dict[str, Any]: - nodes = list(graph.get("nodes", [])) - edges = list(graph.get("edges", [])) - matches = _match_nodes(nodes, query, kind=kind)[:limit] - match_ids = {node["id"] for node in matches} - related_edges = [edge for edge in edges if edge.get("source_id") in match_ids or edge.get("target_id") in match_ids] - related_ids = {edge.get("source_id") for edge in related_edges} | {edge.get("target_id") for edge in related_edges} - related_nodes = [node for node in nodes if node.get("id") in related_ids and node.get("id") not in match_ids] - lines: list[str] = [] - for node in matches: - label = node.get("qualified_name") or node.get("label") or node.get("id") - path = node.get("path") or "" - lines.append(f"- {node.get('table')}: {label} {f'({path})' if path else ''}".strip()) - for edge in related_edges[: max(0, budget // 80)]: - lines.append(f"- {edge.get('type')}: {edge.get('source_id')} -> {edge.get('target_id')}") - text = "\n".join(lines) - if len(text) > budget: - text = text[:budget].rstrip() - payload = { - "query": query, - "profile": profile, - "max_depth": max_depth, - "context": text, - "items": matches, - "related": related_nodes[:limit], - "edge_count": len(related_edges), - } - if include_raw: - payload["raw_edges"] = related_edges - return payload - -def _match_nodes(nodes: list[dict[str, Any]], query: str, kind: str | None = None) -> list[dict[str, Any]]: - terms = [term for term in query.lower().replace("_", " ").replace(".", " ").split() if term] - scored: list[tuple[int, dict[str, Any]]] = [] - for node in nodes: - if kind and node.get("table") != kind and node.get("kind") != kind: - continue - haystack = " ".join( - str(node.get(field, "")) for field in ("id", "label", "qualified_name", "path", "summary", "kind", "table") - ).lower() - score = sum(1 for term in terms if term in haystack) - if score or not terms: - scored.append((score, node)) - return [node for _, node in sorted(scored, key=lambda item: (-item[0], item[1].get("id", "")))] diff --git a/src/codebase_graph/graph_core.py b/src/codebase_graph/graph_core.py deleted file mode 100644 index 1497fd2..0000000 --- a/src/codebase_graph/graph_core.py +++ /dev/null @@ -1,323 +0,0 @@ -from __future__ import annotations - -import argparse -import json -import re -import sys -from dataclasses import dataclass -from pathlib import Path -from typing import Any, Sequence - -from .code_map import CODE_EXTENSIONS, _iter_indexable_files -from .graph_context import build_compact_graph_context -from .ladybug import HashingEmbeddingProvider, LadybugGraphExporter, LadybugGraphStore -from .ontology import schema_payload - -DEFAULT_STATE_DIR = Path(".codebase_graph/graph") -DEFAULT_DB_FILENAME = "knowledge_graph.json" -DEFAULT_STAGING_DIRNAME = "staging" - -@dataclass(slots=True) -class GraphCoreStatus: - source_root: Path - state_dir: Path - database_path: Path - staging_dir: Path - database_exists: bool - stale: bool - source_file_count: int - latest_source_mtime: float | None - database_mtime: float | None - - def as_dict(self) -> dict[str, Any]: - return { - "source_root": str(self.source_root), - "state_dir": str(self.state_dir), - "database_path": str(self.database_path), - "staging_dir": str(self.staging_dir), - "database_exists": self.database_exists, - "stale": self.stale, - "source_file_count": self.source_file_count, - "latest_source_mtime": self.latest_source_mtime, - "database_mtime": self.database_mtime, - "recommended_search_command": "codebase-graph search '' --source-root .", - "recommended_cypher_command": 'codebase-graph cypher "MATCH (n:PythonClass) RETURN n.id, n.label LIMIT 5" --source-root .', - "recommended_schema_command": "codebase-graph schema", - "recommended_context_command": "codebase-graph context '' --source-root .", - } - -class CodebaseGraph: - def __init__( - self, - source_root: str | Path = ".", - state_dir: str | Path | None = None, - database_path: str | Path | None = None, - staging_dir: str | Path | None = None, - embedding_provider: HashingEmbeddingProvider | None = None, - ) -> None: - self.source_root = Path(source_root) - self.state_dir = Path(state_dir) if state_dir is not None else self.source_root / DEFAULT_STATE_DIR - self.database_path = Path(database_path) if database_path is not None else self.state_dir / DEFAULT_DB_FILENAME - self.staging_dir = Path(staging_dir) if staging_dir is not None else self.state_dir / DEFAULT_STAGING_DIRNAME - self.embedding_provider = embedding_provider or HashingEmbeddingProvider() - - def status(self) -> GraphCoreStatus: - mtimes = _source_mtimes(self.source_root) - database_exists = self.database_path.exists() - database_mtime = self.database_path.stat().st_mtime if database_exists else None - latest_source_mtime = max(mtimes) if mtimes else None - stale = not database_exists or ( - latest_source_mtime is not None and database_mtime is not None and latest_source_mtime > database_mtime - ) - return GraphCoreStatus( - source_root=self.source_root, - state_dir=self.state_dir, - database_path=self.database_path, - staging_dir=self.staging_dir, - database_exists=database_exists, - stale=stale, - source_file_count=len(mtimes), - latest_source_mtime=latest_source_mtime, - database_mtime=database_mtime, - ) - - def materialize(self, overwrite: bool = True) -> dict[str, Any]: - if self.database_path.exists() and not overwrite: - raise ValueError(f"Graph database already exists: {self.database_path}") - export = LadybugGraphExporter(self.source_root, embedding_provider=self.embedding_provider).build_export() - self.staging_dir.mkdir(parents=True, exist_ok=True) - store = LadybugGraphStore(self.database_path) - store.write_export(export) - return {"database_path": str(self.database_path), "summary": export.summary()} - - def ensure_current(self) -> dict[str, Any]: - status = self.status() - if status.stale: - return self.materialize(overwrite=True) - return {"database_path": str(self.database_path), "summary": {"status": "current"}} - - def schema(self) -> dict[str, Any]: - return schema_payload() - - def search(self, query: str, limit: int = 10, refresh: bool = True, reinforce: bool = True) -> dict[str, Any]: - if refresh: - self.ensure_current() - graph = self._read_graph() - terms = _terms(query) - scored: list[tuple[float, dict[str, Any]]] = [] - for node in graph.get("nodes", []): - haystack = _node_text(node) - score = sum(2.0 if term in str(node.get("label", "")).lower() else 1.0 for term in terms if term in haystack) - if score > 0 or not terms: - item = _compact_node(node) - item["score"] = score - scored.append((score, item)) - items = [item for _, item in sorted(scored, key=lambda pair: (-pair[0], pair[1].get("id", "")))[:limit]] - return { - "query": query, - "items": items, - "count": len(items), - "database_path": str(self.database_path), - "retrieval": "lexical_graph", - } - - def context( - self, - query: str, - *, - kind: str | None = None, - profile: str = "dependencies", - limit: int = 3, - max_depth: int = 1, - budget: int = 600, - include_raw: bool = False, - refresh: bool = True, - ) -> dict[str, Any]: - if refresh: - self.ensure_current() - return build_compact_graph_context( - self._read_graph(), - query, - kind=kind, - profile=profile, - limit=limit, - max_depth=max_depth, - budget=budget, - include_raw=include_raw, - ) - - def cypher(self, query: str, parameters: dict[str, Any] | None = None, refresh: bool = True) -> dict[str, Any]: - if refresh: - self.ensure_current() - if not _is_read_only_query(query): - raise ValueError("Only read-only MATCH queries are supported") - graph = self._read_graph() - return _run_simple_match_query(graph, query, parameters or {}) - - def _read_graph(self) -> dict[str, Any]: - return LadybugGraphStore(self.database_path).read_export() - -def main(argv: Sequence[str] | None = None) -> int: - argv = _normalize_global_args(list(sys.argv[1:] if argv is None else argv)) - parser = argparse.ArgumentParser(description="Query or rebuild a generic codebase graph.") - parser.add_argument("--source-root", default=".") - parser.add_argument("--state-dir", default=None) - parser.add_argument("--db-path", default=None) - parser.add_argument("--staging-dir", default=None) - subparsers = parser.add_subparsers(dest="command", required=True) - subparsers.add_parser("status") - materialize_parser = subparsers.add_parser("materialize") - materialize_parser.add_argument("--no-overwrite", action="store_true") - subparsers.add_parser("schema") - search_parser = subparsers.add_parser("search") - search_parser.add_argument("query") - search_parser.add_argument("--limit", type=int, default=10) - search_parser.add_argument("--no-refresh", action="store_true") - context_parser = subparsers.add_parser("context") - context_parser.add_argument("query") - context_parser.add_argument("--kind") - context_parser.add_argument("--profile", default="dependencies") - context_parser.add_argument("--limit", type=int, default=3) - context_parser.add_argument("--max-depth", type=int, default=1) - context_parser.add_argument("--budget", type=int, default=600) - context_parser.add_argument("--include-raw", action="store_true") - context_parser.add_argument("--no-refresh", action="store_true") - cypher_parser = subparsers.add_parser("cypher") - cypher_parser.add_argument("query") - cypher_parser.add_argument("--params-json", default="{}") - cypher_parser.add_argument("--no-refresh", action="store_true") - args = parser.parse_args(argv) - core = CodebaseGraph( - source_root=args.source_root, - state_dir=args.state_dir, - database_path=args.db_path, - staging_dir=args.staging_dir, - ) - if args.command == "status": - payload = core.status().as_dict() - elif args.command == "schema": - payload = core.schema() - elif args.command == "materialize": - payload = core.materialize(overwrite=not args.no_overwrite) - elif args.command == "search": - payload = core.search(args.query, limit=args.limit, refresh=not args.no_refresh) - elif args.command == "context": - payload = core.context( - args.query, - kind=args.kind, - profile=args.profile, - limit=args.limit, - max_depth=args.max_depth, - budget=args.budget, - include_raw=args.include_raw, - refresh=not args.no_refresh, - ) - elif args.command == "cypher": - params = json.loads(args.params_json) - if not isinstance(params, dict): - raise ValueError("--params-json must decode to an object") - payload = core.cypher(args.query, parameters=params, refresh=not args.no_refresh) - else: - parser.error(f"unsupported command: {args.command}") - print(json.dumps(payload, indent=2, sort_keys=True)) - return 0 - - -def _normalize_global_args(argv: list[str]) -> list[str]: - value_flags = {"--source-root", "--state-dir", "--db-path", "--staging-dir"} - extracted: list[str] = [] - remaining: list[str] = [] - index = 0 - while index < len(argv): - item = argv[index] - if item in value_flags and index + 1 < len(argv): - extracted.extend([item, argv[index + 1]]) - index += 2 - continue - remaining.append(item) - index += 1 - return extracted + remaining - -def _source_mtimes(source_root: Path) -> list[float]: - mtimes: list[float] = [] - paths = { - *_iter_indexable_files(source_root, CODE_EXTENSIONS), - *_iter_indexable_files(source_root, {".md", ".txt", ".rst", ".toml"}, case_insensitive_suffixes=True), - } - for path in sorted(paths): - try: - mtimes.append(path.stat().st_mtime) - except OSError: - continue - return mtimes - -def _terms(query: str) -> list[str]: - return [term for term in re.split(r"[^a-zA-Z0-9_]+", query.lower()) if term] - -def _node_text(node: dict[str, Any]) -> str: - return " ".join( - str(node.get(field, "")) for field in ("id", "table", "label", "kind", "path", "qualified_name", "summary") - ).lower() - -def _compact_node(node: dict[str, Any]) -> dict[str, Any]: - return { - "id": node.get("id"), - "table": node.get("table"), - "label": node.get("label"), - "kind": node.get("kind"), - "path": node.get("path"), - "qualified_name": node.get("qualified_name"), - "line_start": node.get("line_start"), - "summary": node.get("summary"), - } - -def _is_read_only_query(query: str) -> bool: - lowered = query.strip().lower() - return lowered.startswith("match ") and not any( - token in lowered for token in (" create ", " merge ", " delete ", " set ", " drop ", " copy ", " load ") - ) - -def _run_simple_match_query(graph: dict[str, Any], query: str, parameters: dict[str, Any]) -> dict[str, Any]: - match = re.search(r"MATCH\s*\(\s*(\w+)\s*:\s*(\w+)\s*\)", query, flags=re.IGNORECASE) - if not match: - raise ValueError("Only simple MATCH (n:Label) queries are supported") - variable, table = match.group(1), match.group(2) - where = re.search(r"WHERE\s+(.+?)\s+RETURN", query, flags=re.IGNORECASE | re.DOTALL) - return_match = re.search(r"RETURN\s+(.+?)(?:\s+LIMIT\s+(\d+))?\s*$", query, flags=re.IGNORECASE | re.DOTALL) - if not return_match: - raise ValueError("Query must include RETURN") - columns = [column.strip() for column in return_match.group(1).split(",")] - limit = int(return_match.group(2) or 100) - rows: list[dict[str, Any]] = [] - for node in graph.get("nodes", []): - if node.get("table") != table: - continue - if where and not _where_matches(node, variable, where.group(1), parameters): - continue - row: dict[str, Any] = {} - for column in columns: - if column == variable: - row[column] = node - elif column.startswith(f"{variable}."): - field = column.split(".", 1)[1] - row[column] = node.get(field) - else: - row[column] = node.get(column) - rows.append(row) - if len(rows) >= limit: - break - return {"query": query, "columns": columns, "rows": rows, "count": len(rows), "database_path": graph.get("database_path")} - -def _where_matches(node: dict[str, Any], variable: str, expression: str, parameters: dict[str, Any]) -> bool: - equals = re.match(rf"{re.escape(variable)}\.(\w+)\s*=\s*(.+)$", expression.strip()) - if not equals: - return True - field, raw_value = equals.group(1), equals.group(2).strip() - if raw_value.startswith("$"): - expected = parameters.get(raw_value[1:]) - else: - expected = raw_value.strip("\'\"") - return node.get(field) == expected - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/src/codebase_graph/ladybug.py b/src/codebase_graph/ladybug.py deleted file mode 100644 index 01123c3..0000000 --- a/src/codebase_graph/ladybug.py +++ /dev/null @@ -1,77 +0,0 @@ -from __future__ import annotations - -import hashlib -import json -import math -from dataclasses import dataclass -from pathlib import Path -from typing import Any, Protocol - -from .production import GraphExport, ProductionGraphBuilder - -DEFAULT_EMBEDDING_DIMENSIONS = 384 - -class LadybugUnavailableError(RuntimeError): - pass - -class EmbeddingProvider(Protocol): - dimensions: int - - def embed(self, text: str) -> list[float]: - ... - -class HashingEmbeddingProvider: - def __init__(self, dimensions: int = DEFAULT_EMBEDDING_DIMENSIONS) -> None: - self.dimensions = dimensions - - def embed(self, text: str) -> list[float]: - vector = [0.0] * self.dimensions - for token in text.lower().split(): - digest = hashlib.sha1(token.encode("utf-8")).digest() - index = int.from_bytes(digest[:4], "big") % self.dimensions - vector[index] += 1.0 - norm = math.sqrt(sum(value * value for value in vector)) or 1.0 - return [value / norm for value in vector] - -@dataclass(slots=True) -class LadybugGraphExport: - export: GraphExport - embedding_dimensions: int = DEFAULT_EMBEDDING_DIMENSIONS - - def as_dict(self) -> dict[str, Any]: - payload = self.export.as_dict() - payload["embedding_dimensions"] = self.embedding_dimensions - return payload - - def summary(self) -> dict[str, Any]: - payload = self.export.summary() - payload["embedding_dimensions"] = self.embedding_dimensions - return payload - -class LadybugGraphExporter: - def __init__(self, repo_root: str | Path = ".", embedding_provider: EmbeddingProvider | None = None) -> None: - self.repo_root = Path(repo_root) - self.embedding_provider = embedding_provider or HashingEmbeddingProvider() - - def build_export(self) -> LadybugGraphExport: - export = ProductionGraphBuilder(self.repo_root).build_export() - return LadybugGraphExport(export, int(self.embedding_provider.dimensions)) - -class LadybugGraphStore: - def __init__(self, db_path: str | Path) -> None: - self.db_path = Path(db_path) - - def write_export(self, export: LadybugGraphExport) -> None: - self.db_path.parent.mkdir(parents=True, exist_ok=True) - self.db_path.write_text(json.dumps(export.as_dict(), indent=2, sort_keys=True), encoding="utf-8") - - def read_export(self) -> dict[str, Any]: - if not self.db_path.exists(): - return {"ontology": "", "metadata": {}, "nodes": [], "edges": []} - return json.loads(self.db_path.read_text(encoding="utf-8")) - - def ensure_schema(self, embedding_dimensions: int = DEFAULT_EMBEDDING_DIMENSIONS) -> None: - self.db_path.parent.mkdir(parents=True, exist_ok=True) - - def copy_from_staging(self, staging: Any) -> None: - raise LadybugUnavailableError("Staging copy is not implemented for the JSON-backed base store.") diff --git a/src/codebase_graph/markdown.py b/src/codebase_graph/markdown.py deleted file mode 100644 index f2e6deb..0000000 --- a/src/codebase_graph/markdown.py +++ /dev/null @@ -1,32 +0,0 @@ -from __future__ import annotations - -import re -from typing import Any - -FRONTMATTER_RE = re.compile(r"^---\s*\n(.*?)\n---\s*\n", re.DOTALL) -WIKI_LINK_RE = re.compile(r"\[\[([^\]#|]+)(?:#[^\]|]+)?(?:\|([^\]]+))?\]\]") - -def normalize_slug(value: str) -> str: - slug = re.sub(r"[^a-zA-Z0-9]+", "-", value.strip().lower()).strip("-") - return slug or "untitled" - -def extract_wiki_links(markdown: str) -> list[str]: - return [normalize_slug(match.group(1)) for match in WIKI_LINK_RE.finditer(markdown)] - -def parse_markdown(content: str) -> tuple[dict[str, Any], str]: - match = FRONTMATTER_RE.match(content) - if not match: - return {}, content - frontmatter: dict[str, Any] = {} - for line in match.group(1).splitlines(): - if ":" not in line: - continue - key, value = line.split(":", 1) - frontmatter[key.strip()] = value.strip().strip('"') - return frontmatter, content[match.end():] - -def plain_text(markdown: str) -> str: - text = WIKI_LINK_RE.sub(lambda match: match.group(2) or match.group(1), markdown) - text = re.sub(r"`{1,3}[^`]*`{1,3}", " ", text) - text = re.sub(r"[#*_>\-]+", " ", text) - return re.sub(r"\s+", " ", text).strip() diff --git a/src/codebase_graph/ontology.py b/src/codebase_graph/ontology.py deleted file mode 100644 index 320ec88..0000000 --- a/src/codebase_graph/ontology.py +++ /dev/null @@ -1,79 +0,0 @@ -from __future__ import annotations - -ONTOLOGY_NAME = "codebase_graph_v1" - -NODE_TABLES = ( - "Project", - "Repository", - "File", - "DocumentationSource", - "PythonModule", - "PythonClass", - "PythonFunction", - "PythonMethod", - "Import", - "Call", - "Dependency", - "EntryPoint", - "Test", - "Risk", - "Verification", -) - -EDGE_NODE_TABLES = ( - "Contains", - "Imports", - "Calls", - "DependsOn", - "Defines", - "CoveredBy", - "RoutesTo", - "Describes", - "Produces", -) - -TABLE_COLUMNS: dict[str, tuple[str, ...]] = { - table: ( - "id", - "label", - "kind", - "path", - "qualified_name", - "module_name", - "line_start", - "line_end", - "summary", - "metadata", - ) - for table in NODE_TABLES -} - -VECTOR_INDEXES: tuple[tuple[str, str, str], ...] = () -FTS_INDEXES: tuple[tuple[str, str, tuple[str, ...]], ...] = ( - ("File", "idx_file_text", ("label", "path", "summary")), - ("PythonClass", "idx_python_class_text", ("label", "qualified_name", "summary")), - ("PythonFunction", "idx_python_function_text", ("label", "qualified_name", "summary")), - ("PythonMethod", "idx_python_method_text", ("label", "qualified_name", "summary")), - ("DocumentationSource", "idx_documentation_source_text", ("label", "path", "summary")), -) - -def schema_payload() -> dict[str, object]: - return { - "ontology": ONTOLOGY_NAME, - "node_tables": list(NODE_TABLES), - "edge_tables": list(EDGE_NODE_TABLES), - "table_columns": {name: list(columns) for name, columns in TABLE_COLUMNS.items()}, - "vector_indexes": [ - {"table": table, "index": index, "column": column} - for table, index, column in VECTOR_INDEXES - ], - "fts_indexes": [ - {"table": table, "index": index, "columns": list(columns)} - for table, index, columns in FTS_INDEXES - ], - "examples": [ - "MATCH (n:PythonClass) RETURN n.id, n.label, n.qualified_name LIMIT 5", - "MATCH (n:File) RETURN n.path, n.summary LIMIT 10", - "MATCH (n:EntryPoint) RETURN n.label, n.kind, n.path LIMIT 5", - ], - } diff --git a/src/codebase_graph/production.py b/src/codebase_graph/production.py deleted file mode 100644 index 2d61278..0000000 --- a/src/codebase_graph/production.py +++ /dev/null @@ -1,245 +0,0 @@ -from __future__ import annotations - -import hashlib -import json -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - -try: - import tomllib -except ModuleNotFoundError: # pragma: no cover - py310 fallback - import tomli as tomllib # type: ignore[no-redef] - -from .code_map import CodebaseGraphBuilder, _iter_indexable_files -from .document_layers import LogicalChunker -from .markdown import parse_markdown, plain_text -from .ontology import ONTOLOGY_NAME -from .verification import summarize_verification_run - -@dataclass(slots=True) -class GraphExport: - nodes: list[dict[str, Any]] = field(default_factory=list) - edges: list[dict[str, Any]] = field(default_factory=list) - metadata: dict[str, Any] = field(default_factory=dict) - - def as_dict(self) -> dict[str, Any]: - return {"ontology": ONTOLOGY_NAME, "metadata": self.metadata, "nodes": self.nodes, "edges": self.edges} - - def summary(self) -> dict[str, Any]: - node_counts: dict[str, int] = {} - edge_counts: dict[str, int] = {} - for node in self.nodes: - node_counts[node.get("table", "Unknown")] = node_counts.get(node.get("table", "Unknown"), 0) + 1 - for edge in self.edges: - edge_counts[edge.get("type", "Unknown")] = edge_counts.get(edge.get("type", "Unknown"), 0) + 1 - return { - "ontology": ONTOLOGY_NAME, - "node_count": len(self.nodes), - "edge_count": len(self.edges), - "node_counts": node_counts, - "edge_counts": edge_counts, - } - -class ProductionGraphBuilder: - def __init__(self, repo_root: str | Path = ".") -> None: - self.repo_root = Path(repo_root) - self.nodes: dict[str, dict[str, Any]] = {} - self.edges: dict[str, dict[str, Any]] = {} - - def build_export(self) -> GraphExport: - project_name = self._project_name() - project_id = _id("project", project_name) - repository_id = _id("repository", str(self.repo_root.resolve())) - self._node("Project", project_id, project_name, "project", path=".") - self._node("Repository", repository_id, self.repo_root.name, "repository", path=str(self.repo_root)) - self._edge("Contains", project_id, repository_id, "project_repository") - self._add_codebase(repository_id) - self._add_documentation(repository_id) - self._add_dependencies(repository_id) - self._add_entry_points(repository_id) - self._add_verification_sources(repository_id) - return GraphExport( - nodes=sorted(self.nodes.values(), key=lambda item: (item.get("table", ""), item.get("id", ""))), - edges=sorted(self.edges.values(), key=lambda item: item.get("id", "")), - metadata={"project_name": project_name, "source_root": str(self.repo_root)}, - ) - - def _add_codebase(self, repository_id: str) -> None: - code_map = CodebaseGraphBuilder(self.repo_root).build() - for file in code_map.files: - file_node_id = file.id - self._node( - "File", - file_node_id, - Path(file.path).name, - "python_file", - path=file.path, - module_name=file.module_name, - summary=file.summary, - metadata={"line_count": file.line_count, "language": file.language}, - ) - self._edge("Contains", repository_id, file_node_id, "repository_file") - module_id = _id("module", file.module_name or file.path) - self._node( - "PythonModule", - module_id, - file.module_name or file.path, - "python_module", - path=file.path, - module_name=file.module_name, - qualified_name=file.module_name, - summary=file.summary, - ) - self._edge("Defines", file_node_id, module_id, "file_module") - for imported in file.imports: - import_id = _id("import", f"{file.path}:{imported}") - self._node("Import", import_id, imported, "python_import", path=file.path, qualified_name=imported) - self._edge("Imports", module_id, import_id, "module_import") - for call in file.calls: - call_id = _id("call", f"{file.path}:{call}") - self._node("Call", call_id, call, "python_call", path=file.path, qualified_name=call) - self._edge("Calls", module_id, call_id, "module_call") - for symbol in file.symbols: - table = { - "python_class": "PythonClass", - "python_function": "PythonFunction", - "python_method": "PythonMethod", - }.get(symbol.kind, "PythonFunction") - self._node( - table, - symbol.id, - symbol.label, - symbol.kind, - path=symbol.path, - module_name=symbol.module_name, - qualified_name=symbol.qualified_name, - line_start=symbol.line_start, - line_end=symbol.line_end, - summary=symbol.summary, - metadata={"decorators": symbol.decorators, "bases": symbol.bases}, - ) - self._edge("Defines", module_id, symbol.id, "module_symbol") - - def _add_documentation(self, repository_id: str) -> None: - chunker = LogicalChunker(max_chars=1200) - for path in _iter_documentation_files(self.repo_root): - rel_path = path.relative_to(self.repo_root).as_posix() - content = path.read_text(encoding="utf-8", errors="replace") - _, body = parse_markdown(content) if path.suffix.lower() == ".md" else ({}, content) - summary = plain_text(body)[:500] - doc_id = _id("doc", rel_path) - self._node( - "DocumentationSource", - doc_id, - path.name, - "documentation_source", - path=rel_path, - summary=summary, - metadata={"chunks": [_chunk_as_dict(chunk) for chunk in chunker.chunk(body)[:5]]}, - ) - self._edge("Describes", repository_id, doc_id, "repository_documentation") - - def _add_dependencies(self, repository_id: str) -> None: - pyproject = self.repo_root / "pyproject.toml" - if not pyproject.exists(): - return - payload = tomllib.loads(pyproject.read_text(encoding="utf-8")) - project = payload.get("project", {}) if isinstance(payload, dict) else {} - dependencies = project.get("dependencies", []) if isinstance(project, dict) else [] - for dependency in dependencies: - name = str(dependency).split(";", 1)[0].strip() - dep_id = _id("dependency", name) - self._node("Dependency", dep_id, name, "python_dependency", path="pyproject.toml", summary=str(dependency)) - self._edge("DependsOn", repository_id, dep_id, "declared_dependency") - - def _add_entry_points(self, repository_id: str) -> None: - pyproject = self.repo_root / "pyproject.toml" - if not pyproject.exists(): - return - payload = tomllib.loads(pyproject.read_text(encoding="utf-8")) - scripts = payload.get("project", {}).get("scripts", {}) if isinstance(payload, dict) else {} - if not isinstance(scripts, dict): - return - for name, target in sorted(scripts.items()): - entry_id = _id("entry", f"script:{name}") - self._node("EntryPoint", entry_id, name, "console_script", path="pyproject.toml", qualified_name=str(target)) - self._edge("Produces", repository_id, entry_id, "repository_entry_point") - - def _add_verification_sources(self, repository_id: str) -> None: - for directory in (self.repo_root / ".codebase_graph" / "verification_runs", self.repo_root / "verification"): - if not directory.exists(): - continue - for path in sorted(directory.glob("*.json")): - try: - payload = json.loads(path.read_text(encoding="utf-8")) - except json.JSONDecodeError: - continue - command = str(payload.get("command", "")) - output = str(payload.get("output", "")) - exit_code = payload.get("exit_code") - summary = summarize_verification_run(command, output, exit_code if isinstance(exit_code, int) else None) - verification_id = _id("verification", path.relative_to(self.repo_root).as_posix()) - self._node( - "Verification", - verification_id, - summary["tool"], - "verification_run", - path=path.relative_to(self.repo_root).as_posix(), - summary=summary["summary"], - metadata=summary, - ) - self._edge("Produces", repository_id, verification_id, "repository_verification") - - def _project_name(self) -> str: - pyproject = self.repo_root / "pyproject.toml" - if pyproject.exists(): - try: - payload = tomllib.loads(pyproject.read_text(encoding="utf-8")) - name = payload.get("project", {}).get("name") - if name: - return str(name) - except Exception: - pass - return self.repo_root.name - - def _node(self, table: str, node_id: str, label: str, kind: str, **fields: Any) -> None: - existing = self.nodes.get(node_id, {}) - node = { - "id": node_id, - "table": table, - "label": label, - "kind": kind, - "path": fields.pop("path", ""), - "qualified_name": fields.pop("qualified_name", ""), - "module_name": fields.pop("module_name", ""), - "line_start": fields.pop("line_start", None), - "line_end": fields.pop("line_end", None), - "summary": fields.pop("summary", ""), - "metadata": fields.pop("metadata", {}), - } - node.update(fields) - existing.update({key: value for key, value in node.items() if value not in (None, "", {})}) - self.nodes[node_id] = existing or node - - def _edge(self, edge_type: str, source_id: str, target_id: str, kind: str, **fields: Any) -> None: - edge_id = _id("edge", f"{edge_type}:{source_id}:{target_id}:{kind}") - self.edges[edge_id] = { - "id": edge_id, - "type": edge_type, - "kind": kind, - "source_id": source_id, - "target_id": target_id, - "metadata": fields, - } - - -def _chunk_as_dict(chunk: Any) -> dict[str, Any]: - return {"id": chunk.id, "heading": chunk.heading, "text": chunk.text, "ordinal": chunk.ordinal} - -def _iter_documentation_files(root: Path) -> list[Path]: - suffixes = {".md", ".txt", ".rst"} - return _iter_indexable_files(root, suffixes, case_insensitive_suffixes=True) - -def _id(prefix: str, value: str) -> str: - return f"{prefix}:{hashlib.sha1(value.encode('utf-8')).hexdigest()[:20]}" diff --git a/src/codebase_graph/production_queries.py b/src/codebase_graph/production_queries.py deleted file mode 100644 index d08097d..0000000 --- a/src/codebase_graph/production_queries.py +++ /dev/null @@ -1,40 +0,0 @@ -from __future__ import annotations - -from collections import Counter -from typing import Any - -from .ontology import schema_payload - -def graph_schema() -> dict[str, Any]: - return schema_payload() - -def graph_query(core: Any, query: str, parameters: dict[str, Any] | None = None) -> dict[str, Any]: - return core.cypher(query, parameters=parameters or {}) - -def graph_coverage(core: Any) -> dict[str, Any]: - core.ensure_current() - graph = core._read_graph() - counts = Counter(node.get("table", "Unknown") for node in graph.get("nodes", [])) - return {"node_counts": dict(counts), "node_count": sum(counts.values())} - -def repository_analysis(core: Any) -> dict[str, Any]: - search = core.search("project repository python module class function", limit=20) - return {"retrieval": search.get("retrieval"), "items": search.get("items", []), "count": search.get("count", 0)} - -def risk_report(core: Any) -> dict[str, Any]: - result = core.cypher("MATCH (n:Risk) RETURN n.id, n.label, n.summary LIMIT 25") - return {"items": result.get("rows", []), "count": result.get("count", 0)} - -def task_report(core: Any) -> dict[str, Any]: - return {"items": [], "count": 0} - -def artifact_by_id(core: Any, artifact_id: str) -> dict[str, Any] | None: - core.ensure_current() - for node in core._read_graph().get("nodes", []): - if node.get("id") == artifact_id: - return node - return None - -def explain_decision(core: Any, decision_id: str) -> dict[str, Any]: - artifact = artifact_by_id(core, decision_id) - return {"id": decision_id, "artifact": artifact, "explanation": artifact.get("summary") if artifact else ""} diff --git a/src/codebase_graph/question_query_registry.py b/src/codebase_graph/question_query_registry.py deleted file mode 100644 index d7c47ce..0000000 --- a/src/codebase_graph/question_query_registry.py +++ /dev/null @@ -1,124 +0,0 @@ -from __future__ import annotations - -import argparse -import json -from collections.abc import Sequence -from dataclasses import dataclass -from typing import Any, Mapping - -PHASE_ARCHITECTURE_UNDERSTANDING = "architecture_understanding" -PHASE_CHANGE_PREPARATION = "change_preparation" -PHASE_BREAKING_CHANGE_PREPARATION = "breaking_change_preparation" - -@dataclass(frozen=True, slots=True) -class EngineeringQuestionQuery: - id: str - question: str - intent: str - phase: str - query: str - required_params: tuple[str, ...] = () - result_shape: tuple[str, ...] = () - tags: tuple[str, ...] = () - - def validate_params(self, params: Mapping[str, Any]) -> None: - missing = [name for name in self.required_params if not params.get(name)] - if missing: - raise ValueError(f"Missing required params for {self.id}: {', '.join(missing)}") - - def run(self, core: Any, **params: Any) -> dict[str, Any]: - self.validate_params(params) - return core.cypher(self.query, parameters=dict(params)) - -_QUERIES: tuple[EngineeringQuestionQuery, ...] = ( - EngineeringQuestionQuery( - id="se.architecture.entrypoints.v1", - question="What are the main CLI or package entry points?", - intent="Map runtime ingress points before architecture reasoning.", - phase=PHASE_ARCHITECTURE_UNDERSTANDING, - query="MATCH (n:EntryPoint) RETURN n.id, n.label, n.kind, n.path, n.qualified_name LIMIT 50", - result_shape=("id", "label", "kind", "path", "qualified_name"), - tags=("architecture", "entrypoints", "runtime"), - ), - EngineeringQuestionQuery( - id="se.change.tests_for_artifact.v1", - question="What test artifacts exist for this path or symbol?", - intent="Identify existing tests for target path or symbol before edits.", - phase=PHASE_CHANGE_PREPARATION, - query="MATCH (n:Test) RETURN n.id, n.label, n.kind, n.path, n.qualified_name LIMIT 50", - required_params=("path", "symbol"), - result_shape=("id", "label", "kind", "path", "qualified_name"), - tags=("change", "tests", "coverage"), - ), - EngineeringQuestionQuery( - id="se.breaking.consumers_of_contract.v1", - question="Who are the consumers of the old behavior?", - intent="Find direct consumers of a contract before introducing breaking changes.", - phase=PHASE_BREAKING_CHANGE_PREPARATION, - query="MATCH (n:Call) RETURN n.id, n.label, n.kind, n.path, n.qualified_name LIMIT 50", - required_params=("contract_id",), - result_shape=("id", "label", "kind", "path", "qualified_name"), - tags=("breaking-change", "consumers", "contract"), - ), -) - -def list_engineering_question_queries(phase: str | None = None) -> list[EngineeringQuestionQuery]: - return [query for query in _QUERIES if phase is None or query.phase == phase] - -def get_engineering_question_query(query_id: str) -> EngineeringQuestionQuery: - for query in _QUERIES: - if query.id == query_id: - return query - raise KeyError(query_id) - -def main(argv: Sequence[str] | None = None) -> int: - parser = argparse.ArgumentParser(description="List or run versioned engineering question queries.") - parser.add_argument("--source-root", default=".") - parser.add_argument("--state-dir", default=None) - parser.add_argument("--db-path", default=None) - parser.add_argument("--staging-dir", default=None) - subparsers = parser.add_subparsers(dest="command", required=True) - list_parser = subparsers.add_parser("list") - list_parser.add_argument("--phase") - run_parser = subparsers.add_parser("run") - run_parser.add_argument("query_id") - run_parser.add_argument("--params-json", default="{}") - run_parser.add_argument("--no-refresh", action="store_true") - args = parser.parse_args(argv) - if args.command == "list": - queries = list_engineering_question_queries(phase=args.phase) - payload = {"count": len(queries), "items": [_query_as_dict(query) for query in queries]} - elif args.command == "run": - from .graph_core import CodebaseGraph - - params = json.loads(args.params_json) - if not isinstance(params, dict): - raise ValueError("--params-json must decode to an object") - core = CodebaseGraph( - source_root=args.source_root, - state_dir=args.state_dir, - database_path=args.db_path, - staging_dir=args.staging_dir, - ) - if not args.no_refresh: - core.ensure_current() - question = get_engineering_question_query(args.query_id) - payload = {"question": _query_as_dict(question), "result": question.run(core, **params)} - else: - parser.error(f"unsupported command: {args.command}") - print(json.dumps(payload, indent=2, sort_keys=True)) - return 0 - -def _query_as_dict(query: EngineeringQuestionQuery) -> dict[str, Any]: - return { - "id": query.id, - "question": query.question, - "intent": query.intent, - "phase": query.phase, - "required_params": list(query.required_params), - "result_shape": list(query.result_shape), - "tags": list(query.tags), - } - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/src/codebase_graph/verification.py b/src/codebase_graph/verification.py deleted file mode 100644 index 11aea62..0000000 --- a/src/codebase_graph/verification.py +++ /dev/null @@ -1,32 +0,0 @@ -from __future__ import annotations - -import re -from typing import Any - -def summarize_verification_run(command: str, output: str, exit_code: int | None = None) -> dict[str, Any]: - status = "passed" if exit_code == 0 else "failed" if exit_code else "unknown" - return { - "command": command, - "status": status, - "exit_code": exit_code, - "summary": _compact_output(output), - "tool": _tool_name_from_command(command), - } - -def redact_verification_text(text: str) -> str: - text = re.sub(r"(?i)(api[_-]?key|token|secret|password)=\S+", r"\1=", text) - return text - -def _compact_output(output: str, limit: int = 1200) -> str: - cleaned = redact_verification_text(output).strip() - if len(cleaned) <= limit: - return cleaned - return f"{cleaned[:limit].rstrip()}..." - -def _tool_name_from_command(command: str) -> str: - parts = command.strip().split() - if not parts: - return "unknown" - if parts[0] in {"python", "python3"} and len(parts) > 2 and parts[1] == "-m": - return parts[2] - return parts[0] diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index fdcbc1f..0000000 --- a/tests/conftest.py +++ /dev/null @@ -1,9 +0,0 @@ -from __future__ import annotations - -import sys -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -SRC = ROOT / "src" -if str(SRC) not in sys.path: - sys.path.insert(0, str(SRC)) diff --git a/tests/test_cli.py b/tests/test_cli.py deleted file mode 100644 index ecc9726..0000000 --- a/tests/test_cli.py +++ /dev/null @@ -1,20 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path - -from codebase_graph.cli import main - -FIXTURE = Path(__file__).parent / "fixtures" / "sample_project" - -def test_cli_status_schema_and_search(tmp_path: Path, capsys) -> None: - state_dir = tmp_path / "graph" - assert main(["--source-root", str(FIXTURE), "--state-dir", str(state_dir), "status"]) == 0 - status = json.loads(capsys.readouterr().out) - assert status["stale"] is True - assert main(["--source-root", str(FIXTURE), "--state-dir", str(state_dir), "schema"]) == 0 - schema = json.loads(capsys.readouterr().out) - assert schema["ontology"] == "codebase_graph_v1" - assert main(["--source-root", str(FIXTURE), "--state-dir", str(state_dir), "search", "SampleService"]) == 0 - search = json.loads(capsys.readouterr().out) - assert search["count"] >= 1 diff --git a/tests/test_graph_core.py b/tests/test_graph_core.py deleted file mode 100644 index 98c0f3d..0000000 --- a/tests/test_graph_core.py +++ /dev/null @@ -1,51 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -from codebase_graph import CodebaseGraph -from codebase_graph.code_map import MAX_INDEXED_FILE_BYTES - -FIXTURE = Path(__file__).parent / "fixtures" / "sample_project" - -def test_import_and_status_defaults(tmp_path: Path) -> None: - graph = CodebaseGraph(source_root=FIXTURE, state_dir=tmp_path / "graph") - status = graph.status().as_dict() - assert status["database_exists"] is False - assert status["stale"] is True - assert status["database_path"].endswith("knowledge_graph.json") - -def test_materialize_schema_search_context_and_cypher(tmp_path: Path) -> None: - graph = CodebaseGraph(source_root=FIXTURE, state_dir=tmp_path / "graph") - materialized = graph.materialize() - assert materialized["summary"]["ontology"] == "codebase_graph_v1" - assert materialized["summary"]["node_count"] > 0 - schema = graph.schema() - assert schema["ontology"] == "codebase_graph_v1" - search = graph.search("SampleService", limit=5) - assert any(item["label"] == "SampleService" for item in search["items"]) - context = graph.context("SampleService", budget=500) - assert "SampleService" in context["context"] - cypher = graph.cypher("MATCH (n:PythonClass) RETURN n.id, n.label, n.qualified_name LIMIT 5") - assert cypher["count"] == 1 - assert cypher["rows"][0]["n.label"] == "SampleService" - -def test_status_and_materialize_ignore_non_indexable_files(tmp_path: Path) -> None: - repo = tmp_path / "repo" - repo.mkdir() - (repo / "pyproject.toml").write_text('[project]\nname = "filter-fixture"\nversion = "0.1.0"\n') - (repo / "included.py").write_text("class IncludedClass:\n pass\n") - (repo / "README.MD").write_text("# Fixture\n\nUppercase markdown suffix should stay indexable.\n") - - (repo / "build").mkdir() - (repo / "build" / "generated.py").write_text("class IgnoredBuildClass:\n pass\n") - (repo / "oversized.py").write_text("#" * (MAX_INDEXED_FILE_BYTES + 1)) - (repo / "payload.json").write_text('{"ignored": true}\n') - - graph = CodebaseGraph(source_root=repo, state_dir=tmp_path / "graph") - assert graph.status().source_file_count == 3 - - graph.materialize() - assert any(item["label"] == "IncludedClass" for item in graph.search("IncludedClass", limit=5)["items"]) - assert graph.search("IgnoredBuildClass", limit=5)["count"] == 0 - assert graph.search("oversized", limit=5)["count"] == 0 - assert graph.search("payload", limit=5)["count"] == 0 diff --git a/tests/test_question_query_registry.py b/tests/test_question_query_registry.py deleted file mode 100644 index cf0468f..0000000 --- a/tests/test_question_query_registry.py +++ /dev/null @@ -1,61 +0,0 @@ -from __future__ import annotations - -import json - -import pytest - -import codebase_graph.graph_core as graph_core -from codebase_graph.question_query_registry import ( - PHASE_ARCHITECTURE_UNDERSTANDING, - PHASE_BREAKING_CHANGE_PREPARATION, - get_engineering_question_query, - list_engineering_question_queries, - main, -) - -class _StubCore: - def cypher(self, query, parameters=None): - return {"query": query, "parameters": parameters or {}} - -def test_registry_lists_queries_and_filters_by_phase() -> None: - all_queries = list_engineering_question_queries() - assert len(all_queries) >= 3 - architecture_queries = list_engineering_question_queries(phase=PHASE_ARCHITECTURE_UNDERSTANDING) - assert architecture_queries - assert all(query.phase == PHASE_ARCHITECTURE_UNDERSTANDING for query in architecture_queries) - -def test_get_query_by_id_and_run() -> None: - query = get_engineering_question_query("se.breaking.consumers_of_contract.v1") - assert query.phase == PHASE_BREAKING_CHANGE_PREPARATION - response = query.run(_StubCore(), contract_id="contract:example") - assert response["parameters"]["contract_id"] == "contract:example" - -def test_required_params_are_validated() -> None: - query = get_engineering_question_query("se.change.tests_for_artifact.v1") - with pytest.raises(ValueError): - query.run(_StubCore(), path="src/package", symbol="") - -def test_cli_lists_queries_by_phase(capsys) -> None: - exit_code = main(["list", "--phase", PHASE_ARCHITECTURE_UNDERSTANDING]) - assert exit_code == 0 - payload = json.loads(capsys.readouterr().out) - assert payload["count"] >= 1 - assert all(item["phase"] == PHASE_ARCHITECTURE_UNDERSTANDING for item in payload["items"]) - -def test_cli_runs_query_with_params(monkeypatch, capsys) -> None: - class _FakeGraphCore: - def __init__(self, **kwargs): - self.kwargs = kwargs - - def ensure_current(self): - return {"ok": True} - - def cypher(self, query, parameters=None): - return {"query": query, "parameters": parameters or {}} - - monkeypatch.setattr(graph_core, "CodebaseGraph", _FakeGraphCore) - exit_code = main(["run", "se.breaking.consumers_of_contract.v1", "--params-json", '{"contract_id": "api:contract"}']) - assert exit_code == 0 - payload = json.loads(capsys.readouterr().out) - assert payload["question"]["id"] == "se.breaking.consumers_of_contract.v1" - assert payload["result"]["parameters"] == {"contract_id": "api:contract"} From da3001f9f6d83bffda50c4d73f73e3c96a3f9896 Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Wed, 20 May 2026 16:54:30 +0930 Subject: [PATCH 02/53] feat: add initial module structure --- pyproject.toml | 6 +- src/cli/__init__.py | 1 + src/core/__init__.py | 1 + src/extract/__init__.py | 1 + src/ingest/__init__.py | 1 + src/memory/__init__.py | 1 + src/ontology/__init__.py | 47 +++ src/ontology/ontology.py | 755 +++++++++++++++++++++++++++++++++++++ src/reasoning/__init__.py | 1 + src/retrieval/__init__.py | 1 + src/storage/__init__.py | 1 + src/tests/__init__.py | 1 + src/tests/test_ontology.py | 120 ++++++ 13 files changed, 933 insertions(+), 4 deletions(-) create mode 100644 src/cli/__init__.py create mode 100644 src/core/__init__.py create mode 100644 src/extract/__init__.py create mode 100644 src/ingest/__init__.py create mode 100644 src/memory/__init__.py create mode 100644 src/ontology/__init__.py create mode 100644 src/ontology/ontology.py create mode 100644 src/reasoning/__init__.py create mode 100644 src/retrieval/__init__.py create mode 100644 src/storage/__init__.py create mode 100644 src/tests/__init__.py create mode 100644 src/tests/test_ontology.py diff --git a/pyproject.toml b/pyproject.toml index fad2c19..2a666e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,9 +25,6 @@ ladybug = ["real_ladybug"] parquet = ["pyarrow"] dev = ["pytest", "ruff"] -[project.scripts] -codebase-graph = "codebase_graph.cli:main" - [tool.setuptools.packages.find] where = ["src"] @@ -36,4 +33,5 @@ line-length = 120 target-version = "py310" [tool.pytest.ini_options] -testpaths = ["tests"] +pythonpath = ["src"] +testpaths = ["src/tests"] diff --git a/src/cli/__init__.py b/src/cli/__init__.py new file mode 100644 index 0000000..9a82187 --- /dev/null +++ b/src/cli/__init__.py @@ -0,0 +1 @@ +"""Command-line entry points.""" diff --git a/src/core/__init__.py b/src/core/__init__.py new file mode 100644 index 0000000..a146e0a --- /dev/null +++ b/src/core/__init__.py @@ -0,0 +1 @@ +"""Public API, models, and protocols for the code memory graph.""" diff --git a/src/extract/__init__.py b/src/extract/__init__.py new file mode 100644 index 0000000..f30f46f --- /dev/null +++ b/src/extract/__init__.py @@ -0,0 +1 @@ +"""Code entity and relation extraction.""" diff --git a/src/ingest/__init__.py b/src/ingest/__init__.py new file mode 100644 index 0000000..4e29ce6 --- /dev/null +++ b/src/ingest/__init__.py @@ -0,0 +1 @@ +"""Repository, documentation, issue, and tool-output ingestion.""" diff --git a/src/memory/__init__.py b/src/memory/__init__.py new file mode 100644 index 0000000..7b38a52 --- /dev/null +++ b/src/memory/__init__.py @@ -0,0 +1 @@ +"""Memory store, recall, update, and consolidation workflows.""" diff --git a/src/ontology/__init__.py b/src/ontology/__init__.py new file mode 100644 index 0000000..dbb9244 --- /dev/null +++ b/src/ontology/__init__.py @@ -0,0 +1,47 @@ +"""Language-neutral code graph ontology.""" + +from .ontology import ( + COMMON_NODE_FIELDS, + CONTEXT_PROFILES, + EDGE_FIELDS, + ONTOLOGY_NAME, + ONTOLOGY_VERSION, + NODE_TYPES, + PARSER_NODE_MAPPINGS, + QUERY_HELPERS, + RELATION_TYPES, + SEARCH_INDEXES, + FieldSpec, + NodeTypeSpec, + ParserNodeMappingSpec, + QueryHelperSpec, + RelationTypeSpec, + get_node_type, + get_relation_type, + node_type_names, + relation_type_names, + schema_payload, +) + +__all__ = [ + "COMMON_NODE_FIELDS", + "CONTEXT_PROFILES", + "EDGE_FIELDS", + "ONTOLOGY_NAME", + "ONTOLOGY_VERSION", + "NODE_TYPES", + "PARSER_NODE_MAPPINGS", + "QUERY_HELPERS", + "RELATION_TYPES", + "SEARCH_INDEXES", + "FieldSpec", + "NodeTypeSpec", + "ParserNodeMappingSpec", + "QueryHelperSpec", + "RelationTypeSpec", + "get_node_type", + "get_relation_type", + "node_type_names", + "relation_type_names", + "schema_payload", +] diff --git a/src/ontology/ontology.py b/src/ontology/ontology.py new file mode 100644 index 0000000..c8db2a2 --- /dev/null +++ b/src/ontology/ontology.py @@ -0,0 +1,755 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +ONTOLOGY_NAME = "code_graph_tree_sitter_v1" +ONTOLOGY_VERSION = "1.0.0" + + +@dataclass(frozen=True, slots=True) +class FieldSpec: + name: str + value_type: str + description: str + required: bool = False + + def as_dict(self) -> dict[str, Any]: + return { + "name": self.name, + "type": self.value_type, + "description": self.description, + "required": self.required, + } + + +@dataclass(frozen=True, slots=True) +class NodeTypeSpec: + name: str + description: str + fields: tuple[FieldSpec, ...] = () + parser_node_types: tuple[str, ...] = () + constraints: tuple[str, ...] = () + + def as_dict(self) -> dict[str, Any]: + return { + "name": self.name, + "description": self.description, + "fields": [field.as_dict() for field in self.fields], + "parser_node_types": list(self.parser_node_types), + "constraints": list(self.constraints), + } + + +@dataclass(frozen=True, slots=True) +class RelationTypeSpec: + name: str + source_types: tuple[str, ...] + target_types: tuple[str, ...] + description: str + fields: tuple[FieldSpec, ...] = () + constraints: tuple[str, ...] = () + + def as_dict(self) -> dict[str, Any]: + return { + "name": self.name, + "source_types": list(self.source_types), + "target_types": list(self.target_types), + "description": self.description, + "fields": [field.as_dict() for field in self.fields], + "constraints": list(self.constraints), + } + + +@dataclass(frozen=True, slots=True) +class ParserNodeMappingSpec: + name: str + parser_node_types: tuple[str, ...] + captures: tuple[str, ...] + target_node_types: tuple[str, ...] + relation_types: tuple[str, ...] + description: str + context_rule: str = "" + + def as_dict(self) -> dict[str, Any]: + return { + "name": self.name, + "parser_node_types": list(self.parser_node_types), + "captures": list(self.captures), + "target_node_types": list(self.target_node_types), + "relation_types": list(self.relation_types), + "description": self.description, + "context_rule": self.context_rule, + } + + +@dataclass(frozen=True, slots=True) +class QueryHelperSpec: + name: str + description: str + query: str + parameters: tuple[str, ...] = () + returns: tuple[str, ...] = () + + def as_dict(self) -> dict[str, Any]: + return { + "name": self.name, + "description": self.description, + "query": self.query, + "parameters": list(self.parameters), + "returns": list(self.returns), + } + + +COMMON_NODE_FIELDS = ( + FieldSpec("id", "string", "Stable unique node identifier.", True), + FieldSpec("label", "string", "Short human-readable node label.", True), + FieldSpec("kind", "string", "Ontology-specific subtype or parser-derived role."), + FieldSpec("language", "string", "Source language when the node is code-derived."), + FieldSpec("path", "string", "Repository-relative source path."), + FieldSpec("qualified_name", "string", "Best-effort language-neutral qualified name."), + FieldSpec("scope_id", "string", "Containing lexical or semantic scope id."), + FieldSpec("line_start", "integer", "One-based start line in the source file."), + FieldSpec("line_end", "integer", "One-based end line in the source file."), + FieldSpec("byte_start", "integer", "Zero-based start byte in the source file."), + FieldSpec("byte_end", "integer", "Zero-based end byte in the source file."), + FieldSpec("tree_sitter_node_type", "string", "Raw parser node type that produced this node."), + FieldSpec("capture_name", "string", "Tree-sitter query capture name when available."), + FieldSpec("summary", "string", "Compact text summary used for context assembly."), + FieldSpec("metadata", "json", "Structured extractor-specific details."), +) + +EDGE_FIELDS = ( + FieldSpec("id", "string", "Stable unique relation identifier.", True), + FieldSpec("kind", "string", "Relation subtype or evidence role."), + FieldSpec("source_id", "string", "Source node id.", True), + FieldSpec("target_id", "string", "Target node id.", True), + FieldSpec("confidence", "number", "Extractor confidence between 0 and 1."), + FieldSpec("line_start", "integer", "One-based evidence start line."), + FieldSpec("line_end", "integer", "One-based evidence end line."), + FieldSpec("byte_start", "integer", "Zero-based evidence start byte."), + FieldSpec("byte_end", "integer", "Zero-based evidence end byte."), + FieldSpec("metadata", "json", "Structured relation evidence and resolver details."), +) + + +def _node( + name: str, + description: str, + *, + parser_node_types: tuple[str, ...] = (), + fields: tuple[FieldSpec, ...] = (), + constraints: tuple[str, ...] = (), +) -> NodeTypeSpec: + return NodeTypeSpec( + name=name, + description=description, + fields=COMMON_NODE_FIELDS + fields, + parser_node_types=parser_node_types, + constraints=constraints, + ) + + +NODE_TYPES = ( + _node("Repository", "A version-controlled repository or source tree boundary."), + _node("SourceRoot", "A configured root scanned for source, docs, manifests, and generated evidence."), + _node( + "File", + "A source, manifest, configuration, or documentation file.", + fields=( + FieldSpec("content_hash", "string", "Hash of file content at extraction time."), + FieldSpec("size_bytes", "integer", "File size in bytes at extraction time."), + ), + ), + _node( + "Module", + "A language-level compilation or namespace unit derived from a source file.", + parser_node_types=("module", "program", "source_file", "Module"), + ), + _node( + "ImportDeclaration", + "An import/include/use/require declaration.", + parser_node_types=( + "import_statement", + "import_from_statement", + "import_declaration", + "Import", + "ImportFrom", + ), + fields=(FieldSpec("imported_name", "string", "Imported module, package, symbol, or path."),), + ), + _node( + "ExportDeclaration", + "An exported symbol or module boundary declaration.", + parser_node_types=("export_statement", "export_clause", "export_declaration"), + ), + _node("Symbol", "A named code artifact when the exact semantic subtype is unresolved."), + _node("Scope", "A lexical or semantic boundary for name resolution."), + _node( + "Class", + "A class, struct, trait, interface, enum class, or similar type container.", + parser_node_types=("class_definition", "class_declaration", "struct_item", "ClassDef"), + ), + _node( + "Function", + "A standalone function, lambda with stable name, or callable declaration.", + parser_node_types=("function_definition", "function_declaration", "FunctionDef"), + ), + _node( + "Method", + "A function declared inside a class, trait, component, or object scope.", + parser_node_types=("method_definition", "method_declaration", "FunctionDef"), + ), + _node("Parameter", "A callable parameter.", parser_node_types=("parameter", "typed_parameter", "arg")), + _node("ReturnType", "A callable return type annotation.", parser_node_types=("return_type", "returns")), + _node( + "TypeAnnotation", + "A type annotation attached to a symbol, parameter, assignment, or return value.", + parser_node_types=("type", "type_identifier", "type_annotation", "annotation"), + ), + _node("TypeAlias", "A named alias for a type expression.", parser_node_types=("type_alias", "type_alias_declaration")), + _node("Variable", "A mutable or local named binding.", parser_node_types=("variable_declaration", "Name")), + _node("Constant", "A named binding treated as stable or immutable by convention or syntax."), + _node("ClassAttribute", "A class-level attribute or static field.", parser_node_types=("AnnAssign", "field_declaration")), + _node("InstanceAttribute", "An instance-level attribute or field assignment."), + _node("Property", "A computed or decorated property exposed as an attribute."), + _node("Decorator", "A decorator, annotation, macro, or attribute attached to a declaration."), + _node("CallExpression", "A call, constructor invocation, message send, or macro invocation.", parser_node_types=("call", "Call")), + _node("Assignment", "An assignment, binding, or destructuring declaration.", parser_node_types=("assignment", "Assign", "AnnAssign")), + _node("Reference", "A name or member reference that may resolve to another node.", parser_node_types=("identifier", "Name")), + _node("Literal", "A literal value from source code.", parser_node_types=("string", "integer", "float", "Constant")), + _node("Expression", "A non-literal expression worth preserving for context or reasoning."), + _node("ControlFlowBlock", "A branch, loop, match, switch, or guard block."), + _node("ExceptionFlow", "A raise, throw, try, catch, except, rescue, or finally flow unit."), + _node("APIEndpoint", "A network, RPC, CLI, event, or message endpoint exposed by code."), + _node("Component", "A UI, service, package, or runtime component represented in source."), + _node("Route", "A route pattern, path binding, or router entry."), + _node("Query", "A database, search, analytics, or graph query string/expression."), + _node("SecretRef", "A reference to a secret, credential, token, key, or sensitive environment variable."), + _node( + "Dependency", + "An external package, library, framework, service, or runtime dependency.", + fields=( + FieldSpec("version", "string", "Declared version or version constraint."), + FieldSpec("ecosystem", "string", "Dependency ecosystem such as pypi, npm, cargo, or go."), + ), + ), + _node("DocumentationSource", "A documentation file or generated documentation artifact."), + _node("DocumentationChunk", "A chunk or heading-level section of documentation."), + _node( + "SyntaxCapture", + "Raw parser evidence preserving the concrete syntax node and capture name.", + fields=( + FieldSpec("sexp", "string", "Optional S-expression or compact parse-tree fragment."), + FieldSpec("text", "string", "Optional source text captured for this syntax node."), + ), + ), +) + + +def _relation( + name: str, + source_types: tuple[str, ...], + target_types: tuple[str, ...], + description: str, + *, + constraints: tuple[str, ...] = (), +) -> RelationTypeSpec: + return RelationTypeSpec( + name=name, + source_types=source_types, + target_types=target_types, + description=description, + fields=EDGE_FIELDS, + constraints=constraints, + ) + + +DECLARATION_NODES = ( + "Symbol", + "Class", + "Function", + "Method", + "Parameter", + "ReturnType", + "TypeAnnotation", + "TypeAlias", + "Variable", + "Constant", + "ClassAttribute", + "InstanceAttribute", + "Property", + "Decorator", + "Assignment", + "APIEndpoint", + "Component", + "Route", + "Query", + "SecretRef", +) + +EXPRESSION_NODES = ( + "CallExpression", + "Assignment", + "Reference", + "Literal", + "Expression", + "ControlFlowBlock", + "ExceptionFlow", + "Query", + "SecretRef", +) + +DOCUMENTATION_NODES = ("DocumentationSource", "DocumentationChunk") + +RELATION_TYPES = ( + _relation( + "Contains", + ("Repository", "SourceRoot", "File", "Module", "Scope", "Class", "Function", "Method", "Component"), + ("SourceRoot", "File", "Module", "Scope", *DECLARATION_NODES, *EXPRESSION_NODES, *DOCUMENTATION_NODES), + "Structural containment between repository, files, scopes, declarations, and syntax-derived units.", + ), + _relation( + "Defines", + ("File", "Module", "Scope", "Class", "Function", "Method", "Component"), + DECLARATION_NODES, + "A file, module, scope, or component defines a semantic code node.", + ), + _relation( + "Imports", + ("File", "Module", "Scope"), + ("ImportDeclaration", "Dependency", "Module", "Symbol"), + "A source unit imports, includes, requires, or uses another unit.", + ), + _relation( + "Exports", + ("File", "Module", "Scope", "Component"), + ("ExportDeclaration", *DECLARATION_NODES), + "A source unit exports a declaration or public surface.", + ), + _relation( + "Declares", + ("Module", "Scope", "Class", "Function", "Method", "Assignment"), + DECLARATION_NODES, + "A declaration site introduces a named symbol or subordinate declaration.", + ), + _relation( + "HasScope", + ("File", "Module", *DECLARATION_NODES, *EXPRESSION_NODES), + ("Scope",), + "Connects a node to the lexical or semantic scope used for resolution.", + ), + _relation( + "HasParameter", + ("Function", "Method", "APIEndpoint", "Route", "CallExpression"), + ("Parameter",), + "Connects callables, endpoints, routes, or calls to their parameters or arguments.", + ), + _relation( + "HasReturnType", + ("Function", "Method", "APIEndpoint"), + ("ReturnType",), + "Connects callables or endpoints to their return type node.", + ), + _relation( + "HasTypeAnnotation", + ("Symbol", "Parameter", "ReturnType", "TypeAlias", "Variable", "Constant", "ClassAttribute", "InstanceAttribute"), + ("TypeAnnotation", "Reference", "Literal"), + "Connects a typed code node to its annotation expression.", + ), + _relation( + "Assigns", + ("Assignment", "Variable", "Constant", "ClassAttribute", "InstanceAttribute", "Property"), + ("Variable", "Constant", "ClassAttribute", "InstanceAttribute", "Property", "Literal", "Expression", "CallExpression"), + "Connects an assignment site or assigned symbol to the target or assigned value.", + ), + _relation( + "References", + ("Reference", "Expression", "CallExpression", "Assignment", "ControlFlowBlock", "Query"), + ("Symbol", "Class", "Function", "Method", "Variable", "Constant", "ClassAttribute", "InstanceAttribute", "Property", "Parameter", "Module", "Dependency"), + "A source reference mentions another semantic node without necessarily resolving to it.", + ), + _relation( + "Calls", + ("Function", "Method", "CallExpression", "APIEndpoint", "Route", "Component"), + ("CallExpression", "Function", "Method", "Class", "APIEndpoint"), + "A callable or call expression invokes another callable-like target.", + ), + _relation( + "DecoratedBy", + ("Class", "Function", "Method", "Property", "APIEndpoint", "Route", "Component"), + ("Decorator", "CallExpression", "Reference"), + "A declaration is modified by a decorator, annotation, macro, or framework marker.", + ), + _relation( + "ResolvesTo", + ("Reference", "ImportDeclaration", "CallExpression", "TypeAnnotation", "Decorator"), + ("Symbol", "Module", "Class", "Function", "Method", "Variable", "Constant", "Dependency", "Parameter"), + "A resolver maps a syntactic reference to the best semantic target.", + ), + _relation( + "DependsOn", + ("Repository", "SourceRoot", "File", "Module", "ImportDeclaration", "Dependency", "Component"), + ("Dependency", "Module", "Component", "SecretRef"), + "A repository or code unit depends on an external or internal dependency.", + ), + _relation( + "Documents", + ("DocumentationSource", "DocumentationChunk", "Literal"), + ("Repository", "File", "Module", *DECLARATION_NODES), + "Documentation describes a repository, source unit, or semantic declaration.", + ), + _relation( + "RoutesTo", + ("Route", "APIEndpoint", "Component"), + ("APIEndpoint", "Function", "Method", "Component"), + "A route or component dispatches to an endpoint or handler.", + ), + _relation( + "Exposes", + ("Repository", "Module", "Component", "APIEndpoint", "Route"), + ("APIEndpoint", "Route", "Function", "Method", "Component", "ExportDeclaration"), + "A source unit exposes a public runtime or module surface.", + ), + _relation( + "ExecutesQuery", + ("Function", "Method", "CallExpression", "APIEndpoint", "Component"), + ("Query",), + "A code path executes or constructs a query.", + ), + _relation( + "UsesSecret", + ("Function", "Method", "CallExpression", "Component", "APIEndpoint", "Dependency"), + ("SecretRef",), + "A code path or dependency uses a secret or sensitive configuration value.", + ), + _relation( + "Raises", + ("Function", "Method", "CallExpression", "ControlFlowBlock"), + ("ExceptionFlow",), + "A code path raises or throws an exception flow.", + ), + _relation( + "Handles", + ("Function", "Method", "ControlFlowBlock", "ExceptionFlow"), + ("ExceptionFlow",), + "A code path handles or catches an exception flow.", + ), + _relation( + "DerivedFrom", + (*DECLARATION_NODES, *EXPRESSION_NODES, "Module", "ImportDeclaration", "ExportDeclaration"), + ("SyntaxCapture",), + "A semantic node was derived from a raw parser capture.", + ), + _relation( + "EvidencedBy", + ("Repository", "File", "Module", *DECLARATION_NODES, *EXPRESSION_NODES, "Dependency", *DOCUMENTATION_NODES), + ("SyntaxCapture", "File", "DocumentationChunk"), + "A semantic claim is supported by parser, file, or documentation evidence.", + ), +) + +PARSER_NODE_MAPPINGS = ( + ParserNodeMappingSpec( + "module", + ("module", "program", "source_file", "Module"), + ("module", "source_file"), + ("Module",), + ("Contains", "Defines", "DerivedFrom"), + "Create one Module node per parser root or language namespace root.", + ), + ParserNodeMappingSpec( + "imports", + ("import_statement", "import_from_statement", "import_declaration", "Import", "ImportFrom"), + ("import", "reference.import", "reference.include", "reference.require", "reference.use"), + ("ImportDeclaration",), + ("Imports", "DependsOn", "DerivedFrom"), + "Normalize import-like declarations across languages and attach imported names as metadata.", + ), + ParserNodeMappingSpec( + "exports", + ("export_statement", "export_clause", "export_declaration"), + ("export", "definition.export"), + ("ExportDeclaration",), + ("Exports", "DerivedFrom"), + "Capture public export declarations and declarations marked as exported.", + ), + ParserNodeMappingSpec( + "classes", + ("class_definition", "class_declaration", "struct_item", "interface_declaration", "ClassDef"), + ("definition.class", "definition.struct", "definition.interface"), + ("Class",), + ("Defines", "Declares", "HasScope", "DecoratedBy", "DerivedFrom"), + "Map class-like containers to Class nodes with a child Scope.", + ), + ParserNodeMappingSpec( + "functions_and_methods", + ("function_definition", "function_declaration", "method_definition", "method_declaration", "FunctionDef"), + ("definition.function", "definition.method"), + ("Function", "Method"), + ("Defines", "Declares", "HasScope", "HasParameter", "HasReturnType", "DecoratedBy", "DerivedFrom"), + "Create Function for module-level callables and Method when the callable is enclosed by Class or Component.", + context_rule="enclosing Class or Component changes the target node from Function to Method", + ), + ParserNodeMappingSpec( + "parameters", + ("parameter", "typed_parameter", "default_parameter", "arg"), + ("definition.parameter", "parameter"), + ("Parameter",), + ("HasParameter", "HasTypeAnnotation", "DerivedFrom"), + "Create Parameter nodes for callable parameter declarations.", + ), + ParserNodeMappingSpec( + "return_types", + ("return_type", "type", "type_identifier", "returns"), + ("type.return", "return_type"), + ("ReturnType",), + ("HasReturnType", "HasTypeAnnotation", "References", "DerivedFrom"), + "Capture explicit return type annotations.", + ), + ParserNodeMappingSpec( + "type_annotations", + ("type", "type_identifier", "type_annotation", "annotation", "Name"), + ("type", "type.annotation", "reference.type"), + ("TypeAnnotation",), + ("HasTypeAnnotation", "References", "ResolvesTo", "DerivedFrom"), + "Capture type annotation expressions attached to declarations.", + ), + ParserNodeMappingSpec( + "type_aliases", + ("type_alias", "type_alias_declaration"), + ("definition.type_alias",), + ("TypeAlias",), + ("Defines", "HasTypeAnnotation", "DerivedFrom"), + "Capture named type aliases.", + ), + ParserNodeMappingSpec( + "assignments", + ("assignment", "assignment_expression", "variable_declaration", "Assign", "AnnAssign"), + ("definition.variable", "definition.constant", "assignment"), + ("Assignment", "Variable", "Constant", "ClassAttribute", "InstanceAttribute", "Property"), + ("Defines", "Declares", "Assigns", "HasTypeAnnotation", "DerivedFrom"), + "Normalize assignments; scope, naming convention, and receiver decide variable, constant, or attribute node type.", + ), + ParserNodeMappingSpec( + "decorators", + ("decorator", "attribute_item", "annotation", "Call", "Name"), + ("decorator", "definition.decorator"), + ("Decorator",), + ("DecoratedBy", "Calls", "References", "DerivedFrom"), + "Capture decorators, annotations, macros, or framework markers that modify declarations.", + ), + ParserNodeMappingSpec( + "calls", + ("call", "call_expression", "invocation_expression", "Call"), + ("reference.call", "call"), + ("CallExpression",), + ("Calls", "References", "ResolvesTo", "DerivedFrom"), + "Create call-expression nodes and optionally resolve them to callable targets.", + ), + ParserNodeMappingSpec( + "references", + ("identifier", "field_identifier", "attribute", "Name", "Attribute"), + ("reference", "reference.identifier", "reference.member"), + ("Reference",), + ("References", "ResolvesTo", "DerivedFrom"), + "Capture name and member references before or after semantic resolution.", + ), + ParserNodeMappingSpec( + "literals", + ("string", "integer", "float", "true", "false", "null", "none", "Constant"), + ("literal", "string", "number"), + ("Literal",), + ("Contains", "References", "DerivedFrom"), + "Capture literals that are useful for docs, routes, queries, secrets, or assignment values.", + ), + ParserNodeMappingSpec( + "control_flow", + ("if_statement", "for_statement", "while_statement", "match_statement", "switch_statement"), + ("control_flow",), + ("ControlFlowBlock",), + ("Contains", "References", "DerivedFrom"), + "Capture branch and loop blocks when they affect reasoning or dependency paths.", + ), + ParserNodeMappingSpec( + "exception_flow", + ("try_statement", "except_clause", "catch_clause", "raise_statement", "throw_statement"), + ("exception", "raises", "handles"), + ("ExceptionFlow",), + ("Raises", "Handles", "DerivedFrom"), + "Capture exception raising and handling paths.", + ), + ParserNodeMappingSpec( + "routes_and_endpoints", + ("decorator", "call", "route_declaration", "handler_definition"), + ("entrypoint.api", "route", "endpoint"), + ("APIEndpoint", "Route"), + ("RoutesTo", "Exposes", "DecoratedBy", "DerivedFrom"), + "Create APIEndpoint and Route nodes from framework route declarations or decorated handlers.", + ), + ParserNodeMappingSpec( + "components", + ("class_definition", "function_definition", "jsx_element", "component_declaration"), + ("definition.component", "component"), + ("Component",), + ("Defines", "Contains", "Exposes", "DerivedFrom"), + "Capture UI, service, runtime, or package components when extractor rules identify them.", + ), + ParserNodeMappingSpec( + "queries", + ("string", "template_string", "call", "Call"), + ("query.sql", "query.graph", "query.search"), + ("Query",), + ("ExecutesQuery", "References", "DerivedFrom"), + "Capture query strings or query builder expressions.", + ), + ParserNodeMappingSpec( + "secrets", + ("identifier", "string", "attribute", "Name", "Constant"), + ("secret", "secret.env", "secret.ref"), + ("SecretRef",), + ("UsesSecret", "References", "DerivedFrom"), + "Capture secret-looking names, environment references, keys, and credential handles.", + ), + ParserNodeMappingSpec( + "documentation", + ("comment", "string", "docstring", "DocumentationSource", "DocumentationChunk"), + ("doc", "doc.string", "doc.comment"), + ("DocumentationSource", "DocumentationChunk"), + ("Documents", "EvidencedBy"), + "Capture documentation sources and chunks from docs, comments, and docstrings.", + ), +) + +SEARCH_INDEXES = ( + {"name": "idx_code_symbols", "node_types": ["Symbol", "Class", "Function", "Method", "Variable", "Constant"], "fields": ["label", "qualified_name", "summary"]}, + {"name": "idx_source_units", "node_types": ["Repository", "SourceRoot", "File", "Module"], "fields": ["label", "path", "summary"]}, + {"name": "idx_dependencies", "node_types": ["ImportDeclaration", "Dependency"], "fields": ["label", "qualified_name", "summary"]}, + {"name": "idx_runtime_surface", "node_types": ["APIEndpoint", "Component", "Route", "Query", "SecretRef"], "fields": ["label", "qualified_name", "summary"]}, + {"name": "idx_docs", "node_types": ["DocumentationSource", "DocumentationChunk"], "fields": ["label", "path", "summary"]}, +) + +CONTEXT_PROFILES = { + "brief": { + "description": "Smallest useful context: matched nodes plus direct defining file/module.", + "relations": ["Contains", "Defines", "EvidencedBy"], + "max_depth": 1, + }, + "definitions": { + "description": "Definition-oriented context for symbols and scopes.", + "relations": ["Defines", "Declares", "HasScope", "HasParameter", "HasReturnType", "HasTypeAnnotation"], + "max_depth": 2, + }, + "dependencies": { + "description": "Import and dependency context.", + "relations": ["Imports", "DependsOn", "References", "ResolvesTo"], + "max_depth": 2, + }, + "callgraph": { + "description": "Callable neighborhood for callers, callees, and call expressions.", + "relations": ["Calls", "References", "ResolvesTo"], + "max_depth": 2, + }, + "runtime": { + "description": "Runtime surface context for routes, endpoints, queries, and secrets.", + "relations": ["RoutesTo", "Exposes", "ExecutesQuery", "UsesSecret"], + "max_depth": 2, + }, + "docs": { + "description": "Documentation context connected to code artifacts.", + "relations": ["Documents", "EvidencedBy"], + "max_depth": 1, + }, + "change_impact": { + "description": "Context for likely downstream impact of changing a symbol.", + "relations": ["Defines", "References", "Calls", "RoutesTo", "ExecutesQuery", "UsesSecret", "DependsOn"], + "max_depth": 3, + }, +} + +QUERY_HELPERS = ( + QueryHelperSpec( + "repository_overview", + "List high-level source roots, files, modules, dependencies, and runtime surfaces.", + "MATCH (n) WHERE n:SourceRoot OR n:File OR n:Module OR n:Dependency OR n:APIEndpoint OR n:Component RETURN n.id, n.label, n.path LIMIT 100", + returns=("id", "label", "path"), + ), + QueryHelperSpec( + "symbol_lookup", + "Find declarations by label or qualified name.", + "MATCH (s:Symbol) WHERE s.label = $name OR s.qualified_name = $name RETURN s.id, s.label, s.qualified_name, s.path LIMIT 25", + parameters=("name",), + returns=("id", "label", "qualified_name", "path"), + ), + QueryHelperSpec( + "definition_context", + "Find a named class, function, method, variable, or constant definition.", + "MATCH (d) WHERE d:Class OR d:Function OR d:Method OR d:Variable OR d:Constant RETURN d.id, d.label, d.kind, d.path LIMIT 50", + returns=("id", "label", "kind", "path"), + ), + QueryHelperSpec( + "callgraph_neighborhood", + "Find call expressions and resolved callable targets near a symbol.", + "MATCH (c:CallExpression)-[:ResolvesTo]->(target) RETURN c.id, c.path, target.id, target.label LIMIT 50", + returns=("call_id", "path", "target_id", "target_label"), + ), + QueryHelperSpec( + "dependency_map", + "Inspect imports and dependencies.", + "MATCH (i:ImportDeclaration)-[:DependsOn]->(d:Dependency) RETURN i.id, i.label, d.id, d.label LIMIT 100", + returns=("import_id", "import_label", "dependency_id", "dependency_label"), + ), + QueryHelperSpec( + "runtime_surface", + "Inspect routes, endpoints, executed queries, and secret use.", + "MATCH (r:Route)-[:RoutesTo]->(e:APIEndpoint) RETURN r.id, r.label, e.id, e.label LIMIT 100", + returns=("route_id", "route_label", "endpoint_id", "endpoint_label"), + ), + QueryHelperSpec( + "documentation_context", + "Find documentation chunks connected to code nodes.", + "MATCH (d:DocumentationChunk)-[:Documents]->(n) RETURN d.id, d.label, n.id, n.label LIMIT 50", + returns=("doc_id", "doc_label", "node_id", "node_label"), + ), + QueryHelperSpec( + "unresolved_references", + "Find references that have not been resolved to a semantic target.", + "MATCH (r:Reference) RETURN r.id, r.label, r.path, r.line_start LIMIT 100", + returns=("id", "label", "path", "line_start"), + ), +) + + +def node_type_names() -> tuple[str, ...]: + return tuple(node.name for node in NODE_TYPES) + + +def relation_type_names() -> tuple[str, ...]: + return tuple(relation.name for relation in RELATION_TYPES) + + +def get_node_type(name: str) -> NodeTypeSpec: + for node_type in NODE_TYPES: + if node_type.name == name: + return node_type + raise KeyError(name) + + +def get_relation_type(name: str) -> RelationTypeSpec: + for relation_type in RELATION_TYPES: + if relation_type.name == name: + return relation_type + raise KeyError(name) + + +def schema_payload() -> dict[str, Any]: + return { + "ontology": ONTOLOGY_NAME, + "version": ONTOLOGY_VERSION, + "node_types": [node.as_dict() for node in NODE_TYPES], + "relation_types": [relation.as_dict() for relation in RELATION_TYPES], + "parser_node_mappings": [mapping.as_dict() for mapping in PARSER_NODE_MAPPINGS], + "search_indexes": list(SEARCH_INDEXES), + "context_profiles": CONTEXT_PROFILES, + "query_helpers": [helper.as_dict() for helper in QUERY_HELPERS], + } diff --git a/src/reasoning/__init__.py b/src/reasoning/__init__.py new file mode 100644 index 0000000..d26618a --- /dev/null +++ b/src/reasoning/__init__.py @@ -0,0 +1 @@ +"""Path explanation, causal trace, and context assembly.""" diff --git a/src/retrieval/__init__.py b/src/retrieval/__init__.py new file mode 100644 index 0000000..53a3413 --- /dev/null +++ b/src/retrieval/__init__.py @@ -0,0 +1 @@ +"""Keyword, vector, graph traversal, and ranking retrieval.""" diff --git a/src/storage/__init__.py b/src/storage/__init__.py new file mode 100644 index 0000000..57be53b --- /dev/null +++ b/src/storage/__init__.py @@ -0,0 +1 @@ +"""Storage adapters, schema management, and migrations.""" diff --git a/src/tests/__init__.py b/src/tests/__init__.py new file mode 100644 index 0000000..6f9bec8 --- /dev/null +++ b/src/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for the fresh src module layout.""" diff --git a/src/tests/test_ontology.py b/src/tests/test_ontology.py new file mode 100644 index 0000000..377c6f4 --- /dev/null +++ b/src/tests/test_ontology.py @@ -0,0 +1,120 @@ +from __future__ import annotations + +import json +import re + +from ontology import ( + PARSER_NODE_MAPPINGS, + QUERY_HELPERS, + RELATION_TYPES, + get_node_type, + get_relation_type, + node_type_names, + relation_type_names, + schema_payload, +) + + +def test_schema_payload_is_json_serializable() -> None: + payload = schema_payload() + + encoded = json.dumps(payload, sort_keys=True) + + assert "code_graph_tree_sitter_v1" in encoded + assert payload["node_types"] + assert payload["relation_types"] + + +def test_required_node_types_are_declared() -> None: + names = set(node_type_names()) + + assert { + "Module", + "ImportDeclaration", + "ExportDeclaration", + "Symbol", + "Scope", + "Class", + "Function", + "Method", + "Parameter", + "ReturnType", + "TypeAnnotation", + "TypeAlias", + "Variable", + "Constant", + "ClassAttribute", + "InstanceAttribute", + "Property", + "Decorator", + "CallExpression", + "Assignment", + "Reference", + "Literal", + "Expression", + "ControlFlowBlock", + "ExceptionFlow", + "APIEndpoint", + "Component", + "Route", + "Query", + "SecretRef", + "Repository", + "SourceRoot", + "File", + "Dependency", + "DocumentationSource", + "DocumentationChunk", + "SyntaxCapture", + } <= names + + +def test_declared_relation_endpoints_reference_declared_node_types() -> None: + names = set(node_type_names()) + + for relation in RELATION_TYPES: + assert relation.source_types + assert relation.target_types + assert set(relation.source_types) <= names + assert set(relation.target_types) <= names + + +def test_parser_node_mappings_reference_declared_nodes_and_relations() -> None: + nodes = set(node_type_names()) + relations = set(relation_type_names()) + + for mapping in PARSER_NODE_MAPPINGS: + assert mapping.parser_node_types + assert mapping.target_node_types + assert set(mapping.target_node_types) <= nodes + assert set(mapping.relation_types) <= relations + + +def test_example_parser_shapes_are_covered() -> None: + covered_parser_nodes = {node for mapping in PARSER_NODE_MAPPINGS for node in mapping.parser_node_types} + + assert { + "Module", + "ImportFrom", + "ClassDef", + "FunctionDef", + "AnnAssign", + "Assign", + "Call", + "Name", + "Attribute", + "Constant", + } <= covered_parser_nodes + + +def test_query_helpers_are_read_only() -> None: + forbidden = re.compile(r"\b(CREATE|MERGE|DELETE|SET|DROP|LOAD|COPY)\b", re.IGNORECASE) + + for helper in QUERY_HELPERS: + assert helper.query.lstrip().upper().startswith("MATCH ") + assert not forbidden.search(helper.query) + + +def test_lookup_helpers_return_expected_specs() -> None: + assert get_node_type("Class").name == "Class" + assert get_relation_type("Calls").name == "Calls" From 476fe5cdcc7c90babc1e8fd07dc8459540633bb0 Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Thu, 21 May 2026 07:32:04 +0930 Subject: [PATCH 03/53] fix: correct ontology name --- src/ontology/ontology.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ontology/ontology.py b/src/ontology/ontology.py index c8db2a2..1bacf07 100644 --- a/src/ontology/ontology.py +++ b/src/ontology/ontology.py @@ -3,7 +3,7 @@ from dataclasses import dataclass from typing import Any -ONTOLOGY_NAME = "code_graph_tree_sitter_v1" +ONTOLOGY_NAME = "code_ontology_v1" ONTOLOGY_VERSION = "1.0.0" From b720c4a0c22bb8abe413c73626a76b3f616188ca Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Thu, 21 May 2026 08:30:03 +0930 Subject: [PATCH 04/53] feat: added GraphBuilder - Added `GraphBuilder`, `CaptureRecord`, `GraphBuildResult`, and `ParseBundle` classes to facilitate the construction of an ontology graph from parsed code entities. - Implemented methods for building graphs from both AST-shaped trees and capture records. - Enhanced the ontology schema by including new relation types for imports and exports. - Created unit tests for `GraphBuilder` to ensure correct mapping of Python AST to ontology nodes and relationships. - Updated ontology tests to verify JSON serializability of the schema payload. --- src/core/__init__.py | 4 + src/core/graph.py | 166 ++++++ src/{storage => db}/__init__.py | 0 src/extract/__init__.py | 4 + src/extract/graph_builder.py | 992 ++++++++++++++++++++++++++++++++ src/ontology/ontology.py | 14 +- src/tests/test_graph_builder.py | 118 ++++ src/tests/test_ontology.py | 3 +- 8 files changed, 1298 insertions(+), 3 deletions(-) create mode 100644 src/core/graph.py rename src/{storage => db}/__init__.py (100%) create mode 100644 src/extract/graph_builder.py create mode 100644 src/tests/test_graph_builder.py diff --git a/src/core/__init__.py b/src/core/__init__.py index a146e0a..1d480ea 100644 --- a/src/core/__init__.py +++ b/src/core/__init__.py @@ -1 +1,5 @@ """Public API, models, and protocols for the code memory graph.""" + +from .graph import CodeGraph, GraphEdge, GraphNode + +__all__ = ["CodeGraph", "GraphEdge", "GraphNode"] diff --git a/src/core/graph.py b/src/core/graph.py new file mode 100644 index 0000000..3467e18 --- /dev/null +++ b/src/core/graph.py @@ -0,0 +1,166 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from ontology import ONTOLOGY_NAME, get_relation_type + + +@dataclass(slots=True) +class GraphNode: + id: str + table: str + label: str + kind: str = "" + language: str = "" + path: str = "" + qualified_name: str = "" + scope_id: str = "" + line_start: int | None = None + line_end: int | None = None + byte_start: int | None = None + byte_end: int | None = None + tree_sitter_node_type: str = "" + capture_name: str = "" + summary: str = "" + metadata: dict[str, Any] = field(default_factory=dict) + + def as_dict(self) -> dict[str, Any]: + return { + "id": self.id, + "table": self.table, + "label": self.label, + "kind": self.kind, + "language": self.language, + "path": self.path, + "qualified_name": self.qualified_name, + "scope_id": self.scope_id, + "line_start": self.line_start, + "line_end": self.line_end, + "byte_start": self.byte_start, + "byte_end": self.byte_end, + "tree_sitter_node_type": self.tree_sitter_node_type, + "capture_name": self.capture_name, + "summary": self.summary, + "metadata": self.metadata, + } + + +@dataclass(slots=True) +class GraphEdge: + id: str + type: str + source_id: str + target_id: str + kind: str = "" + confidence: float = 1.0 + line_start: int | None = None + line_end: int | None = None + byte_start: int | None = None + byte_end: int | None = None + metadata: dict[str, Any] = field(default_factory=dict) + + def as_dict(self) -> dict[str, Any]: + return { + "id": self.id, + "type": self.type, + "source_id": self.source_id, + "target_id": self.target_id, + "kind": self.kind, + "confidence": self.confidence, + "line_start": self.line_start, + "line_end": self.line_end, + "byte_start": self.byte_start, + "byte_end": self.byte_end, + "metadata": self.metadata, + } + + +@dataclass(slots=True) +class CodeGraph: + nodes: dict[str, GraphNode] = field(default_factory=dict) + edges: dict[str, GraphEdge] = field(default_factory=dict) + ontology: str = ONTOLOGY_NAME + metadata: dict[str, Any] = field(default_factory=dict) + + def add_node(self, node: GraphNode) -> GraphNode: + existing = self.nodes.get(node.id) + if existing is None: + self.nodes[node.id] = node + return node + _merge_node(existing, node) + return existing + + def add_edge(self, edge: GraphEdge) -> GraphEdge: + self.edges.setdefault(edge.id, edge) + return self.edges[edge.id] + + def nodes_by_type(self, table: str) -> list[GraphNode]: + return [node for node in self.nodes.values() if node.table == table] + + def edges_by_type(self, edge_type: str) -> list[GraphEdge]: + return [edge for edge in self.edges.values() if edge.type == edge_type] + + def as_dict(self) -> dict[str, Any]: + return { + "ontology": self.ontology, + "metadata": self.metadata, + "nodes": [ + node.as_dict() + for node in sorted(self.nodes.values(), key=lambda item: (item.table, item.id)) + ], + "edges": [ + edge.as_dict() + for edge in sorted(self.edges.values(), key=lambda item: (item.type, item.id)) + ], + } + + def summary(self) -> dict[str, Any]: + node_counts: dict[str, int] = {} + edge_counts: dict[str, int] = {} + for node in self.nodes.values(): + node_counts[node.table] = node_counts.get(node.table, 0) + 1 + for edge in self.edges.values(): + edge_counts[edge.type] = edge_counts.get(edge.type, 0) + 1 + return { + "ontology": self.ontology, + "node_count": len(self.nodes), + "edge_count": len(self.edges), + "node_counts": node_counts, + "edge_counts": edge_counts, + } + + def validate_schema(self) -> None: + node_tables = {node.id: node.table for node in self.nodes.values()} + for edge in self.edges.values(): + if edge.source_id not in node_tables: + raise ValueError(f"Relation {edge.id} source is missing: {edge.source_id}") + if edge.target_id not in node_tables: + raise ValueError(f"Relation {edge.id} target is missing: {edge.target_id}") + spec = get_relation_type(edge.type) + source_table = node_tables[edge.source_id] + target_table = node_tables[edge.target_id] + if source_table not in spec.source_types: + raise ValueError(f"{edge.type} cannot start from {source_table}") + if target_table not in spec.target_types: + raise ValueError(f"{edge.type} cannot target {target_table}") + + +def _merge_node(existing: GraphNode, incoming: GraphNode) -> None: + for field_name in ( + "label", + "kind", + "language", + "path", + "qualified_name", + "scope_id", + "tree_sitter_node_type", + "capture_name", + "summary", + ): + if not getattr(existing, field_name) and getattr(incoming, field_name): + setattr(existing, field_name, getattr(incoming, field_name)) + for field_name in ("line_start", "line_end", "byte_start", "byte_end"): + if getattr(existing, field_name) is None and getattr(incoming, field_name) is not None: + setattr(existing, field_name, getattr(incoming, field_name)) + existing.metadata.update(incoming.metadata) diff --git a/src/storage/__init__.py b/src/db/__init__.py similarity index 100% rename from src/storage/__init__.py rename to src/db/__init__.py diff --git a/src/extract/__init__.py b/src/extract/__init__.py index f30f46f..43be001 100644 --- a/src/extract/__init__.py +++ b/src/extract/__init__.py @@ -1 +1,5 @@ """Code entity and relation extraction.""" + +from .graph_builder import CaptureRecord, GraphBuilder, GraphBuildResult, ParseBundle + +__all__ = ["CaptureRecord", "GraphBuilder", "GraphBuildResult", "ParseBundle"] diff --git a/src/extract/graph_builder.py b/src/extract/graph_builder.py new file mode 100644 index 0000000..7d9bbc1 --- /dev/null +++ b/src/extract/graph_builder.py @@ -0,0 +1,992 @@ +from __future__ import annotations + +import hashlib +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from core import CodeGraph, GraphEdge, GraphNode +from ontology import ONTOLOGY_NAME, node_type_names, relation_type_names + + +@dataclass(frozen=True, slots=True) +class CaptureRecord: + capture: str + node: Any + + +@dataclass(frozen=True, slots=True) +class ParseBundle: + language: str + path: str + source_text: str = "" + tree: Any | None = None + captures: Sequence[CaptureRecord | Mapping[str, Any] | tuple[Any, str]] = () + repository_label: str = "repository" + source_root: str = "." + content_hash: str = "" + + +@dataclass(frozen=True, slots=True) +class GraphBuildResult: + nodes: list[dict[str, Any]] + edges: list[dict[str, Any]] + diagnostics: list[str] + unresolved: list[str] + graph: CodeGraph + + def as_dict(self) -> dict[str, Any]: + return { + "nodes": self.nodes, + "edges": self.edges, + "diagnostics": self.diagnostics, + "unresolved": self.unresolved, + "summary": self.graph.summary(), + } + + +@dataclass(frozen=True, slots=True) +class ParserNode: + node_type: str + fields: Mapping[str, Any] + children: tuple[Any, ...] + line_start: int | None = None + line_end: int | None = None + byte_start: int | None = None + byte_end: int | None = None + capture_name: str = "" + text: str = "" + + +@dataclass(frozen=True, slots=True) +class BuildContext: + path: str + language: str + source_text: str + repository_label: str + source_root: str + + +@dataclass(frozen=True, slots=True) +class ScopeFrame: + node_id: str + table: str + label: str + scope_id: str + qualified_name: str + + +class GraphBuilder: + """Build an ontology graph from tree-sitter-shaped parser output. + + The builder deliberately uses duck typing instead of importing tree-sitter. + It accepts dictionaries, Python AST-like objects, and tree-sitter Node-like + objects with ``type``, ``children``, ``start_point``, and ``end_point``. + """ + + def __init__( + self, + *, + default_language: str = "", + repository_label: str = "repository", + source_root: str | Path = ".", + include_syntax_captures: bool = True, + ) -> None: + self.default_language = default_language + self.repository_label = repository_label + self.source_root = Path(source_root).as_posix() + self.include_syntax_captures = include_syntax_captures + self._node_types = set(node_type_names()) + self._relation_types = set(relation_type_names()) + self._graph = CodeGraph() + self._context = BuildContext("", "", "", repository_label, self.source_root) + self._syntax_nodes: dict[int, str] = {} + self._diagnostics: list[str] = [] + self._unresolved: list[str] = [] + + def build_file_graph(self, bundle: ParseBundle) -> GraphBuildResult: + if bundle.captures: + graph = self.build_from_captures( + bundle.captures, + source_path=bundle.path, + language=bundle.language, + source_text=bundle.source_text, + repository_label=bundle.repository_label, + source_root=bundle.source_root, + ) + else: + tree = bundle.tree or {"type": "Module", "children": []} + graph = self.build( + tree, + source_path=bundle.path, + language=bundle.language, + source_text=bundle.source_text, + repository_label=bundle.repository_label, + source_root=bundle.source_root, + ) + if bundle.content_hash: + for node in graph.nodes_by_type("File"): + node.metadata["content_hash"] = bundle.content_hash + return GraphBuildResult( + nodes=graph.as_dict()["nodes"], + edges=graph.as_dict()["edges"], + diagnostics=list(self._diagnostics), + unresolved=list(self._unresolved), + graph=graph, + ) + + def build( + self, + parse_tree: Any, + *, + source_path: str | Path, + language: str | None = None, + source_text: str = "", + repository_label: str | None = None, + source_root: str | Path | None = None, + ) -> CodeGraph: + path = Path(source_path).as_posix() + root = Path(source_root).as_posix() if source_root is not None else self.source_root + repo_label = repository_label or self.repository_label + self._graph = CodeGraph( + ontology=ONTOLOGY_NAME, + metadata={"source_path": path, "language": language or self.default_language, "source_root": root}, + ) + self._context = BuildContext( + path=path, + language=language or self.default_language, + source_text=source_text, + repository_label=repo_label, + source_root=root, + ) + self._syntax_nodes = {} + self._diagnostics = [] + self._unresolved = [] + + repository = self._support_node("Repository", repo_label, repo_label, path="") + source = self._support_node("SourceRoot", root, root, path=root) + file = self._support_node("File", path, Path(path).name, path=path) + self._edge("Contains", repository.id, source.id, "repository_source_root") + self._edge("Contains", source.id, file.id, "source_root_file") + + root_node = self._normalize(parse_tree) + if root_node.node_type in {"Module", "module", "program", "source_file"}: + module = self._semantic_node("Module", root_node, label=_module_label(path), owner=file) + module_scope = self._scope_for(module) + self._edge("Contains", file.id, module.id, "file_module") + self._edge("Contains", module.id, module_scope.id, "module_contains_scope") + self._edge("HasScope", module.id, module_scope.id, "module_scope") + self._traverse(root_node, ScopeFrame(module.id, "Module", module.label, module_scope.id, module.label)) + else: + file_scope = self._scope_for(file) + self._edge("HasScope", file.id, file_scope.id, "file_scope") + self._traverse(root_node, ScopeFrame(file.id, "File", file.label, file_scope.id, file.label)) + + self._graph.validate_schema() + return self._graph + + def build_from_captures( + self, + captures: Iterable[CaptureRecord | Mapping[str, Any] | tuple[Any, str]], + *, + source_path: str | Path, + language: str | None = None, + source_text: str = "", + repository_label: str | None = None, + source_root: str | Path | None = None, + ) -> CodeGraph: + root = { + "type": "Module", + "children": [ + {"type": _capture_node_type(capture), "capture_name": _capture_name(capture), "node": _capture_node(capture)} + for capture in captures + ], + } + return self.build( + root, + source_path=source_path, + language=language, + source_text=source_text, + repository_label=repository_label, + source_root=source_root, + ) + + def _traverse(self, raw_node: Any, owner: ScopeFrame) -> None: + node = self._normalize(raw_node) + syntax_id = self._syntax_capture(node) + next_owner = owner + capture_table = _table_from_capture(node.capture_name, owner) + + if capture_table is not None: + semantic = self._emit_captured_semantic(capture_table, node, owner, syntax_id) + if capture_table in {"Class", "Function", "Method", "Component"}: + scope = self._scope_for(semantic) + self._edge("Contains", semantic.id, scope.id, f"{capture_table.lower()}_contains_scope") + self._edge("HasScope", semantic.id, scope.id, f"{capture_table.lower()}_scope") + next_owner = ScopeFrame(semantic.id, capture_table, semantic.label, scope.id, semantic.qualified_name) + elif node.node_type in {"Module", "module", "program", "source_file"} and owner.table != "Module": + semantic = self._semantic_node("Module", node, label=_module_label(self._context.path), owner_id=owner.node_id) + scope = self._scope_for(semantic) + self._edge("Contains", owner.node_id, semantic.id, "contains_module") + self._edge("Contains", semantic.id, scope.id, "module_contains_scope") + self._edge("HasScope", semantic.id, scope.id, "module_scope") + self._derived_from(semantic.id, syntax_id) + next_owner = ScopeFrame(semantic.id, "Module", semantic.label, scope.id, semantic.qualified_name) + elif node.node_type in IMPORT_NODE_TYPES: + self._emit_import(node, owner, syntax_id) + elif node.node_type in EXPORT_NODE_TYPES: + self._emit_simple_semantic("ExportDeclaration", node, owner, syntax_id) + elif node.node_type in CLASS_NODE_TYPES: + semantic = self._emit_declaration("Class", node, owner, syntax_id) + scope = self._scope_for(semantic) + self._edge("Contains", semantic.id, scope.id, "class_contains_scope") + self._edge("HasScope", semantic.id, scope.id, "class_scope") + next_owner = ScopeFrame(semantic.id, "Class", semantic.label, scope.id, semantic.qualified_name) + self._emit_decorators(node, semantic) + elif node.node_type in FUNCTION_NODE_TYPES: + table = "Method" if owner.table in {"Class", "Component"} else "Function" + semantic = self._emit_declaration(table, node, owner, syntax_id) + scope = self._scope_for(semantic) + self._edge("Contains", semantic.id, scope.id, f"{table.lower()}_contains_scope") + self._edge("HasScope", semantic.id, scope.id, f"{table.lower()}_scope") + next_owner = ScopeFrame(semantic.id, table, semantic.label, scope.id, semantic.qualified_name) + self._emit_parameters(node, semantic) + self._emit_return_type(node, semantic) + self._emit_decorators(node, semantic) + elif node.node_type in ASSIGNMENT_NODE_TYPES: + self._emit_assignment(node, owner, syntax_id) + elif node.node_type in CALL_NODE_TYPES: + self._emit_call(node, owner, syntax_id) + elif node.node_type in REFERENCE_NODE_TYPES: + self._emit_reference(node, owner, syntax_id) + elif node.node_type in LITERAL_NODE_TYPES: + self._emit_simple_semantic("Literal", node, owner, syntax_id) + elif node.node_type in PARAMETER_NODE_TYPES: + self._emit_simple_semantic("Parameter", node, owner, syntax_id) + elif node.node_type in RETURN_TYPE_NODE_TYPES: + self._emit_simple_semantic("ReturnType", node, owner, syntax_id) + elif node.node_type in TYPE_NODE_TYPES: + self._emit_simple_semantic("TypeAnnotation", node, owner, syntax_id) + elif node.node_type in CONTROL_FLOW_NODE_TYPES: + self._emit_simple_semantic("ControlFlowBlock", node, owner, syntax_id) + elif node.node_type in EXCEPTION_FLOW_NODE_TYPES: + self._emit_simple_semantic("ExceptionFlow", node, owner, syntax_id) + + for child in self._semantic_children(node): + self._traverse(child, next_owner) + + def _emit_captured_semantic( + self, + table: str, + node: ParserNode, + owner: ScopeFrame, + syntax_id: str, + ) -> GraphNode: + if table == "ImportDeclaration": + return self._emit_import(node, owner, syntax_id) + if table == "ExportDeclaration": + return self._emit_simple_semantic("ExportDeclaration", node, owner, syntax_id) + if table in {"Class", "Function", "Method"}: + return self._emit_declaration(table, node, owner, syntax_id) + if table == "CallExpression": + return self._emit_call(node, owner, syntax_id) + if table == "Reference": + return self._emit_reference(node, owner, syntax_id) + return self._emit_simple_semantic(table, node, owner, syntax_id) + + def _emit_import(self, node: ParserNode, owner: ScopeFrame, syntax_id: str) -> GraphNode: + imported = _import_label(node) or _label_for(node) + semantic = self._semantic_node( + "ImportDeclaration", + node, + label=imported or node.node_type, + owner_id=owner.node_id, + metadata={"imported_name": imported}, + ) + self._connect_owner(owner, semantic) + self._edge("Imports", owner.node_id, semantic.id, "declares_import") + self._derived_from(semantic.id, syntax_id) + if imported: + dependency = self._support_node("Dependency", imported, imported, path=self._context.path) + self._edge("DependsOn", semantic.id, dependency.id, "import_dependency") + self._edge("EvidencedBy", dependency.id, syntax_id, "parser_evidence") + return semantic + + def _emit_declaration(self, table: str, node: ParserNode, owner: ScopeFrame, syntax_id: str) -> GraphNode: + semantic = self._semantic_node(table, node, owner_id=owner.node_id, owner_qualified_name=owner.qualified_name) + self._connect_owner(owner, semantic) + self._edge("Defines", owner.node_id, semantic.id, f"defines_{table.lower()}") + if owner.table in {"Module", "Scope", "Class", "Function", "Method"}: + self._edge("Declares", owner.node_id, semantic.id, f"declares_{table.lower()}") + self._derived_from(semantic.id, syntax_id) + return semantic + + def _emit_assignment(self, node: ParserNode, owner: ScopeFrame, syntax_id: str) -> GraphNode: + assignment = self._semantic_node("Assignment", node, owner_id=owner.node_id, owner_qualified_name=owner.qualified_name) + self._connect_owner(owner, assignment) + self._derived_from(assignment.id, syntax_id) + + target_label = _assignment_target_label(node) + if target_label: + target_table = _assignment_target_table(target_label, owner, node) + target = self._semantic_node( + target_table, + node, + label=target_label, + owner_id=owner.node_id, + owner_qualified_name=owner.qualified_name, + ) + self._connect_owner(owner, target) + self._edge("Defines", owner.node_id, target.id, f"defines_{target_table.lower()}") + self._edge("Assigns", assignment.id, target.id, "assignment_target") + self._derived_from(target.id, syntax_id) + annotation = _field(node, "annotation") + if annotation is not None: + type_node = self._emit_type_annotation(annotation, target) + self._edge("HasTypeAnnotation", target.id, type_node.id, "assignment_annotation") + + value = _field(node, "value") + if value is not None and _normalized_type(value) in CALL_NODE_TYPES: + call = self._emit_call(self._normalize(value), owner, self._syntax_capture(self._normalize(value))) + self._edge("Assigns", assignment.id, call.id, "assignment_value") + + return assignment + + def _emit_call(self, node: ParserNode, owner: ScopeFrame, syntax_id: str) -> GraphNode: + call = self._semantic_node( + "CallExpression", + node, + label=_call_label(node) or _label_for(node), + owner_id=owner.node_id, + owner_qualified_name=owner.qualified_name, + ) + self._connect_owner(owner, call) + if owner.table in {"Function", "Method", "APIEndpoint", "Route", "Component"}: + self._edge("Calls", owner.node_id, call.id, "body_call") + self._derived_from(call.id, syntax_id) + return call + + def _emit_reference(self, node: ParserNode, owner: ScopeFrame, syntax_id: str) -> GraphNode: + reference = self._semantic_node( + "Reference", + node, + label=_label_for(node), + owner_id=owner.node_id, + owner_qualified_name=owner.qualified_name, + ) + self._connect_owner(owner, reference) + self._derived_from(reference.id, syntax_id) + return reference + + def _emit_simple_semantic(self, table: str, node: ParserNode, owner: ScopeFrame, syntax_id: str) -> GraphNode: + semantic = self._semantic_node( + table, + node, + label=_label_for(node), + owner_id=owner.node_id, + owner_qualified_name=owner.qualified_name, + ) + self._connect_owner(owner, semantic) + self._derived_from(semantic.id, syntax_id) + return semantic + + def _emit_parameters(self, node: ParserNode, callable_node: GraphNode) -> None: + for index, parameter in enumerate(_parameters(node)): + parser_node = self._normalize(parameter) + syntax_id = self._syntax_capture(parser_node) + param_node = self._semantic_node( + "Parameter", + parser_node, + label=_label_for(parser_node) or f"param_{index}", + owner_id=callable_node.id, + owner_qualified_name=callable_node.qualified_name, + ) + self._edge("HasParameter", callable_node.id, param_node.id, "callable_parameter", metadata={"ordinal": index}) + self._derived_from(param_node.id, syntax_id) + annotation = _field(parser_node, "annotation") + if annotation is not None: + type_node = self._emit_type_annotation(annotation, param_node) + self._edge("HasTypeAnnotation", param_node.id, type_node.id, "parameter_annotation") + + def _emit_return_type(self, node: ParserNode, callable_node: GraphNode) -> None: + raw_return = _field(node, "returns") or _field(node, "return_type") + if raw_return is None: + return + return_parser = self._normalize(raw_return) + syntax_id = self._syntax_capture(return_parser) + return_node = self._semantic_node( + "ReturnType", + return_parser, + label=_label_for(return_parser), + owner_id=callable_node.id, + owner_qualified_name=callable_node.qualified_name, + ) + self._edge("HasReturnType", callable_node.id, return_node.id, "callable_return_type") + self._derived_from(return_node.id, syntax_id) + + def _emit_type_annotation(self, raw_node: Any, owner: GraphNode) -> GraphNode: + parser_node = self._normalize(raw_node) + syntax_id = self._syntax_capture(parser_node) + type_node = self._semantic_node( + "TypeAnnotation", + parser_node, + label=_label_for(parser_node), + owner_id=owner.id, + owner_qualified_name=owner.qualified_name, + ) + self._derived_from(type_node.id, syntax_id) + return type_node + + def _emit_decorators(self, node: ParserNode, declaration: GraphNode) -> None: + for raw_decorator in _iter_field_items(node, "decorator_list", "decorators"): + decorator_node = self._normalize(raw_decorator) + syntax_id = self._syntax_capture(decorator_node) + decorator = self._semantic_node( + "Decorator", + decorator_node, + label=_call_label(decorator_node) or _label_for(decorator_node), + owner_id=declaration.id, + owner_qualified_name=declaration.qualified_name, + ) + self._edge("DecoratedBy", declaration.id, decorator.id, "declaration_decorator") + self._derived_from(decorator.id, syntax_id) + + def _connect_owner(self, owner: ScopeFrame, semantic: GraphNode) -> None: + self._edge("Contains", owner.node_id, semantic.id, f"contains_{semantic.table.lower()}") + if owner.scope_id: + self._edge("Contains", owner.scope_id, semantic.id, f"scope_contains_{semantic.table.lower()}") + + def _support_node(self, table: str, stable_key: str, label: str, *, path: str) -> GraphNode: + node = GraphNode( + id=_id(table, stable_key), + table=table, + label=label, + kind=table.lower(), + path=path, + summary=label, + metadata={"canonical_key": stable_key}, + ) + return self._graph.add_node(node) + + def _semantic_node( + self, + table: str, + parser_node: ParserNode, + *, + label: str | None = None, + owner: GraphNode | None = None, + owner_id: str = "", + owner_qualified_name: str = "", + metadata: dict[str, Any] | None = None, + ) -> GraphNode: + if table not in self._node_types: + raise ValueError(f"Unknown ontology node type: {table}") + semantic_label = label or _label_for(parser_node) or table + qualified_name = _qualified_name(owner_qualified_name or (owner.qualified_name if owner else ""), semantic_label) + stable_key = "|".join( + str(value) + for value in ( + self._context.path, + table, + qualified_name, + parser_node.node_type, + parser_node.line_start, + parser_node.byte_start, + semantic_label, + ) + ) + node = GraphNode( + id=_id(table, stable_key), + table=table, + label=semantic_label, + kind=_kind_for(table, parser_node), + language=self._context.language, + path=self._context.path, + qualified_name=qualified_name, + scope_id=owner_id or (owner.id if owner else ""), + line_start=parser_node.line_start, + line_end=parser_node.line_end, + byte_start=parser_node.byte_start, + byte_end=parser_node.byte_end, + tree_sitter_node_type=parser_node.node_type, + capture_name=parser_node.capture_name, + summary=semantic_label, + metadata={"canonical_key": stable_key, **(metadata or {})}, + ) + return self._graph.add_node(node) + + def _scope_for(self, owner: GraphNode) -> GraphNode: + stable_key = f"{self._context.path}|{owner.id}|scope" + scope = GraphNode( + id=_id("Scope", stable_key), + table="Scope", + label=f"{owner.label} scope", + kind=f"{owner.table.lower()}_scope", + language=owner.language, + path=owner.path, + qualified_name=f"{owner.qualified_name or owner.label}.", + scope_id=owner.id, + line_start=owner.line_start, + line_end=owner.line_end, + byte_start=owner.byte_start, + byte_end=owner.byte_end, + summary=f"Scope for {owner.label}", + metadata={"canonical_key": stable_key}, + ) + return self._graph.add_node(scope) + + def _syntax_capture(self, node: ParserNode) -> str: + stable_key = "|".join( + str(value) + for value in (self._context.path, node.node_type, node.line_start, node.byte_start, _label_for(node)) + ) + syntax_id = _id("SyntaxCapture", stable_key) + if not self.include_syntax_captures: + return syntax_id + if id(node) in self._syntax_nodes: + return self._syntax_nodes[id(node)] + syntax = GraphNode( + id=syntax_id, + table="SyntaxCapture", + label=node.capture_name or node.node_type, + kind=node.node_type, + language=self._context.language, + path=self._context.path, + line_start=node.line_start, + line_end=node.line_end, + byte_start=node.byte_start, + byte_end=node.byte_end, + tree_sitter_node_type=node.node_type, + capture_name=node.capture_name, + summary=node.text[:160], + metadata={"canonical_key": stable_key, "fields": sorted(node.fields.keys())}, + ) + self._graph.add_node(syntax) + self._syntax_nodes[id(node)] = syntax_id + return syntax_id + + def _derived_from(self, semantic_id: str, syntax_id: str) -> None: + if self.include_syntax_captures and syntax_id in self._graph.nodes: + self._edge("DerivedFrom", semantic_id, syntax_id, "parser_capture") + + def _edge( + self, + edge_type: str, + source_id: str, + target_id: str, + kind: str, + *, + metadata: dict[str, Any] | None = None, + ) -> GraphEdge: + if edge_type not in self._relation_types: + raise ValueError(f"Unknown ontology relation type: {edge_type}") + edge = GraphEdge( + id=_id("edge", f"{edge_type}|{source_id}|{target_id}|{kind}"), + type=edge_type, + source_id=source_id, + target_id=target_id, + kind=kind, + metadata={"canonical_key": f"{edge_type}|{source_id}|{target_id}|{kind}", **(metadata or {})}, + ) + return self._graph.add_edge(edge) + + def _normalize(self, raw_node: Any) -> ParserNode: + if isinstance(raw_node, ParserNode): + return raw_node + if isinstance(raw_node, Mapping): + nested = raw_node.get("node") + if nested is not None: + nested_node = self._normalize(nested) + return ParserNode( + node_type=str(raw_node.get("type") or nested_node.node_type), + fields={**nested_node.fields, **{key: value for key, value in raw_node.items() if key != "node"}}, + children=nested_node.children, + line_start=nested_node.line_start, + line_end=nested_node.line_end, + byte_start=nested_node.byte_start, + byte_end=nested_node.byte_end, + capture_name=str(raw_node.get("capture_name") or nested_node.capture_name or ""), + text=nested_node.text, + ) + fields = {key: value for key, value in raw_node.items() if key not in DICT_NODE_META_KEYS} + children = tuple(_coerce_children(raw_node)) + return ParserNode( + node_type=str(raw_node.get("type") or raw_node.get("node_type") or raw_node.get("kind") or "unknown"), + fields=fields, + children=children, + line_start=_line(raw_node, "line_start", "start_line"), + line_end=_line(raw_node, "line_end", "end_line"), + byte_start=_line(raw_node, "byte_start", "start_byte"), + byte_end=_line(raw_node, "byte_end", "end_byte"), + capture_name=str(raw_node.get("capture_name") or raw_node.get("capture") or ""), + text=str(raw_node.get("text") or ""), + ) + node_type = getattr(raw_node, "type", "") or type(raw_node).__name__ + fields = _object_fields(raw_node) + return ParserNode( + node_type=str(node_type), + fields=fields, + children=tuple(getattr(raw_node, "children", ()) or _field_children(fields)), + line_start=_point_line(getattr(raw_node, "start_point", None)) or getattr(raw_node, "lineno", None), + line_end=_point_line(getattr(raw_node, "end_point", None)) or getattr(raw_node, "end_lineno", None), + byte_start=getattr(raw_node, "start_byte", None) or getattr(raw_node, "col_offset", None), + byte_end=getattr(raw_node, "end_byte", None) or getattr(raw_node, "end_col_offset", None), + text=_node_text(raw_node), + ) + + def _semantic_children(self, node: ParserNode) -> tuple[Any, ...]: + ignored_fields = {"name", "id", "module", "names", "args", "returns", "return_type", "decorator_list", "decorators"} + children: list[Any] = list(node.children) + for field_name, value in node.fields.items(): + if field_name in ignored_fields: + continue + if _is_parser_like(value): + children.append(value) + elif isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + children.extend(item for item in value if _is_parser_like(item)) + return tuple(children) + + +IMPORT_NODE_TYPES = {"import_statement", "import_from_statement", "import_declaration", "Import", "ImportFrom"} +EXPORT_NODE_TYPES = {"export_statement", "export_clause", "export_declaration"} +CLASS_NODE_TYPES = {"class_definition", "class_declaration", "struct_item", "interface_declaration", "ClassDef"} +FUNCTION_NODE_TYPES = {"function_definition", "function_declaration", "method_definition", "method_declaration", "FunctionDef"} +PARAMETER_NODE_TYPES = {"parameter", "typed_parameter", "default_parameter", "arg"} +RETURN_TYPE_NODE_TYPES = {"return_type", "returns"} +TYPE_NODE_TYPES = {"type", "type_identifier", "type_annotation", "annotation"} +ASSIGNMENT_NODE_TYPES = {"assignment", "assignment_expression", "variable_declaration", "Assign", "AnnAssign"} +CALL_NODE_TYPES = {"call", "call_expression", "invocation_expression", "Call"} +REFERENCE_NODE_TYPES = {"identifier", "field_identifier", "attribute", "Name", "Attribute"} +LITERAL_NODE_TYPES = {"string", "integer", "float", "true", "false", "null", "none", "Constant"} +CONTROL_FLOW_NODE_TYPES = {"if_statement", "for_statement", "while_statement", "match_statement", "switch_statement"} +EXCEPTION_FLOW_NODE_TYPES = {"try_statement", "except_clause", "catch_clause", "raise_statement", "throw_statement"} +DICT_NODE_META_KEYS = { + "type", + "node_type", + "kind", + "children", + "body", + "line_start", + "line_end", + "start_line", + "end_line", + "byte_start", + "byte_end", + "start_byte", + "end_byte", + "capture", + "capture_name", + "text", +} + + +def _capture_node(capture: Mapping[str, Any] | tuple[Any, str]) -> Any: + if isinstance(capture, CaptureRecord): + return capture.node + if isinstance(capture, tuple): + return capture[0] + return capture.get("node") or capture + + +def _capture_name(capture: Mapping[str, Any] | tuple[Any, str]) -> str: + if isinstance(capture, CaptureRecord): + return capture.capture + if isinstance(capture, tuple): + return str(capture[1]) + return str(capture.get("capture_name") or capture.get("capture") or "") + + +def _capture_node_type(capture: Mapping[str, Any] | tuple[Any, str]) -> str: + node = _capture_node(capture) + if isinstance(node, Mapping): + return str(node.get("type") or node.get("node_type") or node.get("kind") or "unknown") + return str(getattr(node, "type", "") or type(node).__name__) + + +def _table_from_capture(capture_name: str, owner: ScopeFrame) -> str | None: + capture = capture_name.lstrip("@") + if not capture: + return None + if capture in {"definition.class", "definition.struct", "definition.interface"}: + return "Class" + if capture == "definition.component" or capture == "component": + return "Component" + if capture == "definition.method": + return "Method" + if capture == "definition.function": + return "Method" if owner.table in {"Class", "Component"} else "Function" + if capture == "definition.parameter" or capture == "parameter": + return "Parameter" + if capture in {"type.return", "return_type"}: + return "ReturnType" + if capture in {"type", "type.annotation", "reference.type"}: + return "TypeAnnotation" + if capture == "definition.type_alias": + return "TypeAlias" + if capture == "definition.constant": + return "Constant" + if capture == "definition.variable": + return "Variable" + if capture in {"decorator", "definition.decorator"}: + return "Decorator" + if capture in {"reference.import", "reference.include", "reference.require", "reference.use", "import"}: + return "ImportDeclaration" + if capture in {"export", "definition.export"}: + return "ExportDeclaration" + if capture in {"reference.call", "call"}: + return "CallExpression" + if capture.startswith("query."): + return "Query" + if capture.startswith("secret."): + return "SecretRef" + if capture in {"entrypoint.api", "endpoint"}: + return "APIEndpoint" + if capture == "route": + return "Route" + if capture.startswith("doc"): + return "DocumentationChunk" + if capture in {"literal", "string", "number"}: + return "Literal" + if capture == "control_flow": + return "ControlFlowBlock" + if capture in {"exception", "raises", "handles"}: + return "ExceptionFlow" + if capture.startswith("reference"): + return "Reference" + return None + + +def _id(prefix: str, value: str) -> str: + return f"{prefix}:{hashlib.sha1(value.encode('utf-8')).hexdigest()[:20]}" + + +def _module_label(path: str) -> str: + stem = path.rsplit(".", 1)[0] + return stem.replace("/", ".") + + +def _qualified_name(owner: str, label: str) -> str: + if not owner or owner == label: + return label + if not label: + return owner + return f"{owner}.{label}" + + +def _kind_for(table: str, node: ParserNode) -> str: + if table == "Method": + return "method" + if table == "Function": + return "function" + if table == "Class": + return "class" + return node.node_type + + +def _field(node: ParserNode, *names: str) -> Any: + for name in names: + if name in node.fields: + return node.fields[name] + return None + + +def _iter_field_items(node: ParserNode, *names: str) -> tuple[Any, ...]: + items: list[Any] = [] + for name in names: + value = node.fields.get(name) + if value is None: + continue + if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + items.extend(value) + else: + items.append(value) + return tuple(items) + + +def _label_for(node: ParserNode) -> str: + for key in ("name", "id", "arg", "attr", "module"): + value = node.fields.get(key) + label = _value_label(value) + if label: + return label + if "value" in node.fields: + return _value_label(node.fields["value"]) + return node.text.strip() or node.node_type + + +def _value_label(value: Any) -> str: + if value is None: + return "" + if isinstance(value, str): + return value + if isinstance(value, (int, float, bool)): + return str(value) + if isinstance(value, Mapping): + if "id" in value: + return str(value["id"]) + if "name" in value: + return str(value["name"]) + if "arg" in value: + return str(value["arg"]) + if "attr" in value: + base = _value_label(value.get("value")) + return f"{base}.{value['attr']}" if base else str(value["attr"]) + if "value" in value: + return _value_label(value["value"]) + if hasattr(value, "id"): + return str(getattr(value, "id")) + if hasattr(value, "name"): + return str(getattr(value, "name")) + if hasattr(value, "arg"): + return str(getattr(value, "arg")) + if hasattr(value, "attr"): + base = _value_label(getattr(value, "value", None)) + return f"{base}.{getattr(value, 'attr')}" if base else str(getattr(value, "attr")) + if hasattr(value, "value"): + return _value_label(getattr(value, "value")) + return "" + + +def _import_label(node: ParserNode) -> str: + module = _value_label(node.fields.get("module")) + names = node.fields.get("names") + imported_names: list[str] = [] + if isinstance(names, Sequence) and not isinstance(names, (str, bytes, bytearray)): + imported_names = [_value_label(name) for name in names if _value_label(name)] + elif names is not None: + imported_names = [_value_label(names)] + if module and imported_names: + return ", ".join(f"{module}.{name}" for name in imported_names) + return module or ", ".join(imported_names) + + +def _call_label(node: ParserNode) -> str: + return _value_label(node.fields.get("func")) or _value_label(node.fields.get("function")) + + +def _assignment_target_label(node: ParserNode) -> str: + target = node.fields.get("target") + targets = node.fields.get("targets") + if target is not None: + return _value_label(target) + if isinstance(targets, Sequence) and not isinstance(targets, (str, bytes, bytearray)) and targets: + return _value_label(targets[0]) + return _value_label(targets) + + +def _assignment_target_table(label: str, owner: ScopeFrame, node: ParserNode) -> str: + if label.isupper(): + return "Constant" + if owner.table == "Class": + return "ClassAttribute" + if "." in label: + return "InstanceAttribute" + if node.node_type == "AnnAssign" and owner.table == "Class": + return "ClassAttribute" + return "Variable" + + +def _parameters(node: ParserNode) -> tuple[Any, ...]: + raw_args = node.fields.get("args") or node.fields.get("parameters") + if raw_args is None: + return () + if isinstance(raw_args, Mapping): + args = raw_args.get("args") or raw_args.get("children") or () + if isinstance(args, Sequence) and not isinstance(args, (str, bytes, bytearray)): + return tuple(args) + if hasattr(raw_args, "args"): + args = getattr(raw_args, "args") + if isinstance(args, Sequence): + return tuple(args) + if isinstance(raw_args, Sequence) and not isinstance(raw_args, (str, bytes, bytearray)): + return tuple(raw_args) + return (raw_args,) + + +def _normalized_type(raw_node: Any) -> str: + if isinstance(raw_node, ParserNode): + return raw_node.node_type + if isinstance(raw_node, Mapping): + return str(raw_node.get("type") or raw_node.get("node_type") or raw_node.get("kind") or "unknown") + return str(getattr(raw_node, "type", "") or type(raw_node).__name__) + + +def _coerce_children(raw_node: Mapping[str, Any]) -> tuple[Any, ...]: + children: list[Any] = [] + for key in ("children", "body"): + value = raw_node.get(key) + if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + children.extend(value) + elif value is not None: + children.append(value) + return tuple(children) + + +def _field_children(fields: Mapping[str, Any]) -> tuple[Any, ...]: + children: list[Any] = [] + for value in fields.values(): + if _is_parser_like(value): + children.append(value) + elif isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + children.extend(item for item in value if _is_parser_like(item)) + return tuple(children) + + +def _object_fields(raw_node: Any) -> Mapping[str, Any]: + if hasattr(raw_node, "_fields"): + return {name: getattr(raw_node, name) for name in getattr(raw_node, "_fields")} + if hasattr(raw_node, "child_by_field_name"): + fields: dict[str, Any] = {} + for name in ("name", "body", "parameters", "return_type", "function", "argument", "left", "right"): + try: + value = raw_node.child_by_field_name(name) + except Exception: + value = None + if value is not None: + fields[name] = value + return fields + return { + key: value + for key, value in vars(raw_node).items() + if not key.startswith("_") and key not in {"children", "type"} + } if hasattr(raw_node, "__dict__") else {} + + +def _is_parser_like(value: Any) -> bool: + if value is None or isinstance(value, (str, bytes, bytearray, int, float, bool)): + return False + if isinstance(value, Mapping): + return any(key in value for key in ("type", "node_type", "kind", "body", "children")) + return hasattr(value, "type") or hasattr(value, "_fields") + + +def _line(raw_node: Mapping[str, Any], *keys: str) -> int | None: + for key in keys: + value = raw_node.get(key) + if isinstance(value, int): + return value + start_point = raw_node.get("start_point") + end_point = raw_node.get("end_point") + if "start" in keys[0] and start_point is not None: + return _point_line(start_point) + if "end" in keys[0] and end_point is not None: + return _point_line(end_point) + return None + + +def _point_line(point: Any) -> int | None: + if point is None: + return None + if isinstance(point, Sequence) and point: + return int(point[0]) + 1 + if hasattr(point, "row"): + return int(getattr(point, "row")) + 1 + return None + + +def _node_text(raw_node: Any) -> str: + text = getattr(raw_node, "text", b"") + if isinstance(text, bytes): + return text.decode("utf-8", errors="replace") + return str(text or "") diff --git a/src/ontology/ontology.py b/src/ontology/ontology.py index 1bacf07..5e23876 100644 --- a/src/ontology/ontology.py +++ b/src/ontology/ontology.py @@ -306,7 +306,17 @@ def _relation( _relation( "Contains", ("Repository", "SourceRoot", "File", "Module", "Scope", "Class", "Function", "Method", "Component"), - ("SourceRoot", "File", "Module", "Scope", *DECLARATION_NODES, *EXPRESSION_NODES, *DOCUMENTATION_NODES), + ( + "SourceRoot", + "File", + "Module", + "Scope", + "ImportDeclaration", + "ExportDeclaration", + *DECLARATION_NODES, + *EXPRESSION_NODES, + *DOCUMENTATION_NODES, + ), "Structural containment between repository, files, scopes, declarations, and syntax-derived units.", ), _relation( @@ -437,7 +447,7 @@ def _relation( ), _relation( "DerivedFrom", - (*DECLARATION_NODES, *EXPRESSION_NODES, "Module", "ImportDeclaration", "ExportDeclaration"), + (*DECLARATION_NODES, *EXPRESSION_NODES, *DOCUMENTATION_NODES, "Module", "ImportDeclaration", "ExportDeclaration"), ("SyntaxCapture",), "A semantic node was derived from a raw parser capture.", ), diff --git a/src/tests/test_graph_builder.py b/src/tests/test_graph_builder.py new file mode 100644 index 0000000..0265723 --- /dev/null +++ b/src/tests/test_graph_builder.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +from extract import CaptureRecord, GraphBuilder, ParseBundle + + +def test_graph_builder_maps_python_ast_shaped_tree_to_ontology() -> None: + parse_tree = { + "type": "Module", + "body": [ + { + "type": "ImportFrom", + "module": "dataclasses", + "names": [{"type": "alias", "name": "dataclass"}], + "line_start": 1, + }, + { + "type": "ClassDef", + "name": "WikiConfig", + "line_start": 5, + "decorator_list": [ + { + "type": "Call", + "func": {"type": "Name", "id": "dataclass"}, + "keywords": [{"type": "keyword", "arg": "slots", "value": {"type": "Constant", "value": True}}], + } + ], + "body": [ + { + "type": "AnnAssign", + "target": {"type": "Name", "id": "vault_dir"}, + "annotation": {"type": "Name", "id": "Path"}, + "value": { + "type": "Call", + "func": {"type": "Name", "id": "Path"}, + "args": [{"type": "Constant", "value": "wiki"}], + }, + "line_start": 7, + }, + { + "type": "FunctionDef", + "name": "raw_dir", + "args": {"type": "arguments", "args": [{"type": "arg", "arg": "self"}]}, + "returns": {"type": "Name", "id": "Path"}, + "decorator_list": [{"type": "Name", "id": "property"}], + "body": [ + { + "type": "Return", + "value": { + "type": "Attribute", + "value": {"type": "Name", "id": "self"}, + "attr": "vault_dir", + }, + } + ], + "line_start": 10, + }, + ], + }, + { + "type": "Assign", + "targets": [{"type": "Name", "id": "PAGE_KINDS"}], + "value": { + "type": "Tuple", + "elts": [ + {"type": "Constant", "value": "sources"}, + {"type": "Constant", "value": "entities"}, + ], + }, + "line_start": 15, + }, + ], + } + + graph = GraphBuilder(default_language="python").build(parse_tree, source_path="wiki_config.py") + + labels_by_type = { + table: {node.label for node in graph.nodes_by_type(table)} + for table in ("ImportDeclaration", "Class", "Method", "ClassAttribute", "Constant", "Decorator") + } + assert "dataclasses.dataclass" in labels_by_type["ImportDeclaration"] + assert "WikiConfig" in labels_by_type["Class"] + assert "raw_dir" in labels_by_type["Method"] + assert "vault_dir" in labels_by_type["ClassAttribute"] + assert "PAGE_KINDS" in labels_by_type["Constant"] + assert {"dataclass", "property"} <= labels_by_type["Decorator"] + assert graph.edges_by_type("DerivedFrom") + assert graph.edges_by_type("HasReturnType") + assert graph.edges_by_type("HasTypeAnnotation") + + +def test_graph_builder_uses_capture_names_as_primary_semantic_signal() -> None: + bundle = ParseBundle( + language="python", + path="api.py", + captures=( + CaptureRecord( + "definition.function", + {"type": "identifier", "text": "handler", "start_byte": 10, "end_byte": 17}, + ), + CaptureRecord( + "reference.call", + {"type": "identifier", "text": "json_response", "start_byte": 24, "end_byte": 37}, + ), + CaptureRecord( + "doc.string", + {"type": "string", "text": "Handle the API request.", "start_byte": 40, "end_byte": 65}, + ), + ), + ) + + result = GraphBuilder(repository_label="sample").build_file_graph(bundle) + graph = result.graph + + assert {node.label for node in graph.nodes_by_type("Function")} == {"handler"} + assert {node.label for node in graph.nodes_by_type("CallExpression")} == {"json_response"} + assert {node.label for node in graph.nodes_by_type("DocumentationChunk")} == {"Handle the API request."} + assert not result.diagnostics + assert not result.unresolved diff --git a/src/tests/test_ontology.py b/src/tests/test_ontology.py index 377c6f4..f03073e 100644 --- a/src/tests/test_ontology.py +++ b/src/tests/test_ontology.py @@ -4,6 +4,7 @@ import re from ontology import ( + ONTOLOGY_NAME, PARSER_NODE_MAPPINGS, QUERY_HELPERS, RELATION_TYPES, @@ -20,7 +21,7 @@ def test_schema_payload_is_json_serializable() -> None: encoded = json.dumps(payload, sort_keys=True) - assert "code_graph_tree_sitter_v1" in encoded + assert ONTOLOGY_NAME in encoded assert payload["node_types"] assert payload["relation_types"] From fe3213eb6f2b3d0e6e386f798ed11c6f8f100548 Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Fri, 22 May 2026 10:36:53 +0930 Subject: [PATCH 05/53] feat: enhance references and relations --- README.md | 2 +- src/extract/graph_builder.py | 292 +++++++++++++++++++++++++++++++- src/ontology/ontology.py | 14 +- src/tests/test_graph_builder.py | 60 +++++++ 4 files changed, 362 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index e4051bc..0157ba0 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # codebaseGraph -`codebase_graph` is a generic project/code knowledge graph engine for Python repositories. It scans a source root, builds a typed graph of files, modules, symbols, imports, calls, dependencies, entry points, and documentation sources, and exposes search, compact context, schema, and read-only query helpers. +`codebaseGraph` is a generic project/code knowledge graph engine for coding repositories. It scans a source root, builds a typed graph of files, modules, symbols, imports, calls, dependencies, entry points, and documentation sources, and exposes search, compact context, schema, and read-only query helpers. ## Install for local development diff --git a/src/extract/graph_builder.py b/src/extract/graph_builder.py index 7d9bbc1..ea3718f 100644 --- a/src/extract/graph_builder.py +++ b/src/extract/graph_builder.py @@ -7,7 +7,7 @@ from typing import Any from core import CodeGraph, GraphEdge, GraphNode -from ontology import ONTOLOGY_NAME, node_type_names, relation_type_names +from ontology import ONTOLOGY_NAME, get_relation_type, node_type_names, relation_type_names @dataclass(frozen=True, slots=True) @@ -102,6 +102,7 @@ def __init__( self._graph = CodeGraph() self._context = BuildContext("", "", "", repository_label, self.source_root) self._syntax_nodes: dict[int, str] = {} + self._symbols_by_name: dict[str, list[str]] = {} self._diagnostics: list[str] = [] self._unresolved: list[str] = [] @@ -161,6 +162,7 @@ def build( source_root=root, ) self._syntax_nodes = {} + self._symbols_by_name = {} self._diagnostics = [] self._unresolved = [] @@ -364,6 +366,9 @@ def _emit_call(self, node: ParserNode, owner: ScopeFrame, syntax_id: str) -> Gra self._connect_owner(owner, call) if owner.table in {"Function", "Method", "APIEndpoint", "Route", "Component"}: self._edge("Calls", owner.node_id, call.id, "body_call") + target = self._emit_reference_edges(call, call.label, kind_prefix="call") + if target is not None: + self._edge_if_allowed("Calls", call.id, target.id, "call_target") self._derived_from(call.id, syntax_id) return call @@ -376,6 +381,7 @@ def _emit_reference(self, node: ParserNode, owner: ScopeFrame, syntax_id: str) - owner_qualified_name=owner.qualified_name, ) self._connect_owner(owner, reference) + self._emit_reference_edges(reference, reference.label, kind_prefix="reference") self._derived_from(reference.id, syntax_id) return reference @@ -388,6 +394,7 @@ def _emit_simple_semantic(self, table: str, node: ParserNode, owner: ScopeFrame, owner_qualified_name=owner.qualified_name, ) self._connect_owner(owner, semantic) + self._emit_contextual_relations(semantic, node, owner, syntax_id) self._derived_from(semantic.id, syntax_id) return semantic @@ -423,6 +430,8 @@ def _emit_return_type(self, node: ParserNode, callable_node: GraphNode) -> None: owner_qualified_name=callable_node.qualified_name, ) self._edge("HasReturnType", callable_node.id, return_node.id, "callable_return_type") + type_node = self._emit_type_annotation(return_parser, return_node) + self._edge("HasTypeAnnotation", return_node.id, type_node.id, "return_type_annotation") self._derived_from(return_node.id, syntax_id) def _emit_type_annotation(self, raw_node: Any, owner: GraphNode) -> GraphNode: @@ -435,6 +444,7 @@ def _emit_type_annotation(self, raw_node: Any, owner: GraphNode) -> GraphNode: owner_id=owner.id, owner_qualified_name=owner.qualified_name, ) + self._emit_reference_edges(type_node, type_node.label, kind_prefix="type_annotation") self._derived_from(type_node.id, syntax_id) return type_node @@ -450,8 +460,107 @@ def _emit_decorators(self, node: ParserNode, declaration: GraphNode) -> None: owner_qualified_name=declaration.qualified_name, ) self._edge("DecoratedBy", declaration.id, decorator.id, "declaration_decorator") + target = self._emit_reference_edges(decorator, decorator.label, kind_prefix="decorator") + if target is not None: + self._edge_if_allowed("Calls", decorator.id, target.id, "decorator_call") self._derived_from(decorator.id, syntax_id) + def _emit_contextual_relations( + self, + semantic: GraphNode, + node: ParserNode, + owner: ScopeFrame, + syntax_id: str, + ) -> None: + table = semantic.table + + if table == "ExportDeclaration": + self._edge_if_allowed("Exports", owner.node_id, semantic.id, "exports_declaration") + target = self._resolve_reference_target(_export_target_label(node) or semantic.label, EXPORT_TARGET_TYPES) + if target is not None and target.id != semantic.id: + self._edge_if_allowed("Exports", owner.node_id, target.id, "exports_symbol") + + if table in DEFINED_CAPTURE_TABLES: + self._edge_if_allowed("Defines", owner.node_id, semantic.id, f"defines_{table.lower()}") + self._edge_if_allowed("Declares", owner.node_id, semantic.id, f"declares_{table.lower()}") + + if table in {"Component", "APIEndpoint", "Route"}: + self._edge_if_allowed("Exposes", owner.node_id, semantic.id, f"exposes_{table.lower()}") + + if table in {"Route", "APIEndpoint"}: + target = self._runtime_target(node, owner, syntax_id) + if target is not None and target.id != semantic.id: + self._edge_if_allowed("RoutesTo", semantic.id, target.id, "runtime_handler") + self._edge_if_allowed("Exposes", semantic.id, target.id, "runtime_surface") + + if table == "Parameter": + self._edge_if_allowed("HasParameter", owner.node_id, semantic.id, "captured_parameter") + annotation = _field(node, "annotation", "type_annotation") + if annotation is not None: + type_node = self._emit_type_annotation(annotation, semantic) + self._edge("HasTypeAnnotation", semantic.id, type_node.id, "parameter_annotation") + + if table == "ReturnType": + self._edge_if_allowed("HasReturnType", owner.node_id, semantic.id, "captured_return_type") + type_node = self._emit_type_annotation(node, semantic) + self._edge("HasTypeAnnotation", semantic.id, type_node.id, "return_type_annotation") + + if table == "TypeAnnotation": + self._edge_if_allowed("HasTypeAnnotation", owner.node_id, semantic.id, "captured_type_annotation") + self._emit_reference_edges(semantic, semantic.label, kind_prefix="type_annotation") + + if table == "TypeAlias": + annotation = _field(node, "annotation", "type_annotation", "value") + if annotation is not None: + type_node = self._emit_type_annotation(annotation, semantic) + self._edge_if_allowed("HasTypeAnnotation", semantic.id, type_node.id, "type_alias_annotation") + + if table == "Decorator": + self._edge_if_allowed("DecoratedBy", owner.node_id, semantic.id, "captured_decorator") + target = self._emit_reference_edges(semantic, semantic.label, kind_prefix="decorator") + if target is not None: + self._edge_if_allowed("Calls", semantic.id, target.id, "decorator_call") + + if table == "Query": + self._edge_if_allowed("ExecutesQuery", owner.node_id, semantic.id, "executes_query") + self._emit_reference_edges(semantic, _query_reference_label(node), kind_prefix="query") + + if table == "SecretRef": + self._edge_if_allowed("UsesSecret", owner.node_id, semantic.id, "uses_secret") + self._emit_reference_edges(semantic, semantic.label, kind_prefix="secret") + + if table in {"DocumentationSource", "DocumentationChunk"}: + self._edge_if_allowed("Documents", semantic.id, owner.node_id, "documents_owner") + self._edge_if_allowed("EvidencedBy", semantic.id, syntax_id, "parser_evidence") + + if table == "ExceptionFlow": + if _is_raise_flow(node): + self._edge_if_allowed("Raises", owner.node_id, semantic.id, "raises_exception") + if _is_handle_flow(node): + self._edge_if_allowed("Handles", owner.node_id, semantic.id, "handles_exception") + + if table == "Reference": + self._emit_reference_edges(semantic, semantic.label, kind_prefix="reference") + + if table == "ControlFlowBlock": + self._emit_reference_edges(semantic, _control_flow_reference_label(node), kind_prefix="control_flow") + + def _emit_reference_edges( + self, + source: GraphNode, + label: str, + *, + kind_prefix: str, + target_tables: set[str] | None = None, + ) -> GraphNode | None: + target = self._resolve_reference_target(label, target_tables) + if target is None or target.id == source.id: + return None + metadata = {"label": label, "resolver": "label"} + self._edge_if_allowed("References", source.id, target.id, f"{kind_prefix}_reference", metadata=metadata) + self._edge_if_allowed("ResolvesTo", source.id, target.id, f"{kind_prefix}_resolution", metadata=metadata) + return target + def _connect_owner(self, owner: ScopeFrame, semantic: GraphNode) -> None: self._edge("Contains", owner.node_id, semantic.id, f"contains_{semantic.table.lower()}") if owner.scope_id: @@ -467,7 +576,9 @@ def _support_node(self, table: str, stable_key: str, label: str, *, path: str) - summary=label, metadata={"canonical_key": stable_key}, ) - return self._graph.add_node(node) + added = self._graph.add_node(node) + self._register_resolvable(added) + return added def _semantic_node( self, @@ -514,7 +625,55 @@ def _semantic_node( summary=semantic_label, metadata={"canonical_key": stable_key, **(metadata or {})}, ) - return self._graph.add_node(node) + added = self._graph.add_node(node) + self._register_resolvable(added) + return added + + def _symbol_node(self, label: str) -> GraphNode | None: + symbol_label = label.strip() + if not symbol_label: + return None + stable_key = f"{self._context.path}|Symbol|{symbol_label}" + node = GraphNode( + id=_id("Symbol", stable_key), + table="Symbol", + label=symbol_label, + kind="symbol_reference", + language=self._context.language, + path=self._context.path, + qualified_name=symbol_label, + summary=symbol_label, + metadata={"canonical_key": stable_key, "resolution": "name_placeholder"}, + ) + added = self._graph.add_node(node) + self._register_resolvable(added) + return added + + def _register_resolvable(self, node: GraphNode) -> None: + if node.table not in RESOLVABLE_NODE_TYPES: + return + keys = {node.label, node.qualified_name, str(node.metadata.get("imported_name") or "")} + for key in keys: + normalized = _symbol_key(key) + if not normalized: + continue + self._symbols_by_name.setdefault(normalized, []) + if node.id not in self._symbols_by_name[normalized]: + self._symbols_by_name[normalized].append(node.id) + + def _resolve_reference_target(self, label: str, target_tables: set[str] | None = None) -> GraphNode | None: + reference_label = label.strip() + if not reference_label: + return None + candidate_labels = (reference_label, reference_label.rsplit(".", 1)[-1]) + for candidate_label in candidate_labels: + for node_id in reversed(self._symbols_by_name.get(_symbol_key(candidate_label), ())): + node = self._graph.nodes.get(node_id) + if node is not None and (target_tables is None or node.table in target_tables): + return node + if target_tables is not None and "Symbol" not in target_tables: + return None + return self._symbol_node(reference_label) def _scope_for(self, owner: GraphNode) -> GraphNode: stable_key = f"{self._context.path}|{owner.id}|scope" @@ -570,6 +729,47 @@ def _derived_from(self, semantic_id: str, syntax_id: str) -> None: if self.include_syntax_captures and syntax_id in self._graph.nodes: self._edge("DerivedFrom", semantic_id, syntax_id, "parser_capture") + def _runtime_target(self, node: ParserNode, owner: ScopeFrame, syntax_id: str) -> GraphNode | None: + label = _runtime_target_label(node) + if label: + target = self._resolve_reference_target(label, RUNTIME_TARGET_TYPES) + if target is not None: + return target + endpoint = self._semantic_node( + "APIEndpoint", + node, + label=label, + owner_id=owner.node_id, + owner_qualified_name=owner.qualified_name, + metadata={"inferred_from": "runtime_target"}, + ) + self._connect_owner(owner, endpoint) + self._edge_if_allowed("Defines", owner.node_id, endpoint.id, "defines_inferred_endpoint") + self._edge_if_allowed("Exposes", owner.node_id, endpoint.id, "exposes_inferred_endpoint") + self._derived_from(endpoint.id, syntax_id) + return endpoint + if owner.table in RUNTIME_TARGET_TYPES: + return self._graph.nodes.get(owner.node_id) + return None + + def _edge_if_allowed( + self, + edge_type: str, + source_id: str, + target_id: str, + kind: str, + *, + metadata: dict[str, Any] | None = None, + ) -> GraphEdge | None: + source = self._graph.nodes.get(source_id) + target = self._graph.nodes.get(target_id) + if source is None or target is None: + return None + spec = get_relation_type(edge_type) + if source.table not in spec.source_types or target.table not in spec.target_types: + return None + return self._edge(edge_type, source_id, target_id, kind, metadata=metadata) + def _edge( self, edge_type: str, @@ -661,6 +861,46 @@ def _semantic_children(self, node: ParserNode) -> tuple[Any, ...]: LITERAL_NODE_TYPES = {"string", "integer", "float", "true", "false", "null", "none", "Constant"} CONTROL_FLOW_NODE_TYPES = {"if_statement", "for_statement", "while_statement", "match_statement", "switch_statement"} EXCEPTION_FLOW_NODE_TYPES = {"try_statement", "except_clause", "catch_clause", "raise_statement", "throw_statement"} +RESOLVABLE_NODE_TYPES = { + "Symbol", + "Module", + "Class", + "Function", + "Method", + "Variable", + "Constant", + "ClassAttribute", + "InstanceAttribute", + "Property", + "Parameter", + "Dependency", + "APIEndpoint", + "Component", +} +EXPORT_TARGET_TYPES = { + "Class", + "Function", + "Method", + "Variable", + "Constant", + "ClassAttribute", + "InstanceAttribute", + "Property", + "APIEndpoint", + "Component", +} +RUNTIME_TARGET_TYPES = {"Function", "Method", "Component", "APIEndpoint"} +DEFINED_CAPTURE_TABLES = { + "APIEndpoint", + "Component", + "Route", + "TypeAlias", + "Variable", + "Constant", + "ClassAttribute", + "InstanceAttribute", + "Property", +} DICT_NODE_META_KEYS = { "type", "node_type", @@ -848,6 +1088,52 @@ def _value_label(value: Any) -> str: return "" +def _symbol_key(label: str) -> str: + return label.strip().lower() + + +def _export_target_label(node: ParserNode) -> str: + for field_name in ("exported", "target", "name", "declaration"): + label = _value_label(node.fields.get(field_name)) + if label: + return label + return _label_for(node) + + +def _runtime_target_label(node: ParserNode) -> str: + for field_name in ("handler", "endpoint", "target", "function", "callback"): + label = _value_label(node.fields.get(field_name)) + if label: + return label + return "" + + +def _query_reference_label(node: ParserNode) -> str: + for field_name in ("table", "collection", "model", "target", "index"): + label = _value_label(node.fields.get(field_name)) + if label: + return label + return "" + + +def _control_flow_reference_label(node: ParserNode) -> str: + for field_name in ("test", "condition", "subject"): + label = _value_label(node.fields.get(field_name)) + if label: + return label + return "" + + +def _is_raise_flow(node: ParserNode) -> bool: + capture = node.capture_name.lstrip("@") + return capture == "raises" or node.node_type in {"raise_statement", "throw_statement"} + + +def _is_handle_flow(node: ParserNode) -> bool: + capture = node.capture_name.lstrip("@") + return capture == "handles" or node.node_type in {"try_statement", "except_clause", "catch_clause"} + + def _import_label(node: ParserNode) -> str: module = _value_label(node.fields.get("module")) names = node.fields.get("names") diff --git a/src/ontology/ontology.py b/src/ontology/ontology.py index 5e23876..7744ca3 100644 --- a/src/ontology/ontology.py +++ b/src/ontology/ontology.py @@ -375,13 +375,23 @@ def _relation( ), _relation( "References", - ("Reference", "Expression", "CallExpression", "Assignment", "ControlFlowBlock", "Query"), + ( + "Reference", + "Expression", + "CallExpression", + "Assignment", + "ControlFlowBlock", + "TypeAnnotation", + "Decorator", + "Query", + "SecretRef", + ), ("Symbol", "Class", "Function", "Method", "Variable", "Constant", "ClassAttribute", "InstanceAttribute", "Property", "Parameter", "Module", "Dependency"), "A source reference mentions another semantic node without necessarily resolving to it.", ), _relation( "Calls", - ("Function", "Method", "CallExpression", "APIEndpoint", "Route", "Component"), + ("Function", "Method", "CallExpression", "Decorator", "APIEndpoint", "Route", "Component"), ("CallExpression", "Function", "Method", "Class", "APIEndpoint"), "A callable or call expression invokes another callable-like target.", ), diff --git a/src/tests/test_graph_builder.py b/src/tests/test_graph_builder.py index 0265723..93a5899 100644 --- a/src/tests/test_graph_builder.py +++ b/src/tests/test_graph_builder.py @@ -1,6 +1,7 @@ from __future__ import annotations from extract import CaptureRecord, GraphBuilder, ParseBundle +from ontology import PARSER_NODE_MAPPINGS def test_graph_builder_maps_python_ast_shaped_tree_to_ontology() -> None: @@ -116,3 +117,62 @@ def test_graph_builder_uses_capture_names_as_primary_semantic_signal() -> None: assert {node.label for node in graph.nodes_by_type("DocumentationChunk")} == {"Handle the API request."} assert not result.diagnostics assert not result.unresolved + + +def test_graph_builder_emits_relation_families_advertised_by_parser_mappings() -> None: + parse_tree = { + "type": "Module", + "body": [ + { + "type": "ImportFrom", + "module": "fastapi", + "names": [{"type": "alias", "name": "APIRouter"}], + }, + {"type": "FunctionDef", "name": "helper", "body": []}, + {"type": "FunctionDef", "name": "auth_required", "body": []}, + { + "type": "FunctionDef", + "name": "list_users", + "args": { + "type": "arguments", + "args": [ + { + "type": "arg", + "arg": "user_id", + "annotation": {"type": "Name", "id": "int"}, + } + ], + }, + "returns": {"type": "Name", "id": "Response"}, + "decorator_list": [{"type": "Name", "id": "auth_required"}], + "body": [ + {"type": "call", "capture_name": "route", "text": "/users", "handler": "list_users"}, + {"type": "Call", "func": {"type": "Name", "id": "helper"}}, + { + "type": "string", + "capture_name": "query.sql", + "text": "SELECT * FROM users", + "table": "users", + }, + {"type": "Name", "capture_name": "secret.env", "id": "DATABASE_URL"}, + { + "type": "Assign", + "targets": [{"type": "Name", "id": "CACHE"}], + "value": {"type": "Call", "func": {"type": "Name", "id": "helper"}}, + }, + {"type": "Name", "capture_name": "reference.identifier", "id": "helper"}, + {"type": "raise_statement", "capture_name": "raises", "name": "ValueError"}, + {"type": "except_clause", "capture_name": "handles", "name": "ValueError"}, + {"type": "docstring", "capture_name": "doc.string", "text": "List users."}, + ], + }, + {"type": "component_declaration", "capture_name": "component", "name": "UserService"}, + {"type": "export_statement", "name": "list_users"}, + ], + } + + graph = GraphBuilder(default_language="python").build(parse_tree, source_path="api.py") + + mapped_relations = {relation for mapping in PARSER_NODE_MAPPINGS for relation in mapping.relation_types} + emitted_relations = set(graph.summary()["edge_counts"]) + assert mapped_relations <= emitted_relations From 850f5c3b500063723277e31a93d6c0f3fe7adf9e Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Fri, 22 May 2026 11:28:59 +0930 Subject: [PATCH 06/53] feat: Implement Code Graph Store and Materialization - Added LadybugCodeGraphStore for managing graph data with LadybugDB. - Implemented methods for schema management, partition replacement, and manifest handling. - Introduced MaterializationManifest and ManifestEntry for tracking materialization state. - Created GraphMaterializer for materializing code graphs from source files. - Developed TreeSitterPythonParser for parsing Python files into graph structures. - Added tests for materialization, schema validation, and graph parsing. --- pyproject.toml | 5 + src/cli/__init__.py | 54 ++++- src/db/__init__.py | 13 ++ src/db/schema.py | 119 ++++++++++ src/db/store.py | 256 +++++++++++++++++++++ src/ingest/__init__.py | 24 ++ src/ingest/materializer.py | 374 +++++++++++++++++++++++++++++++ src/ingest/tree_sitter_parser.py | 250 +++++++++++++++++++++ src/tests/test_materializer.py | 151 +++++++++++++ src/tests/test_schema.py | 119 ++++++++++ 10 files changed, 1364 insertions(+), 1 deletion(-) create mode 100644 src/db/schema.py create mode 100644 src/db/store.py create mode 100644 src/ingest/materializer.py create mode 100644 src/ingest/tree_sitter_parser.py create mode 100644 src/tests/test_materializer.py create mode 100644 src/tests/test_schema.py diff --git a/pyproject.toml b/pyproject.toml index 2a666e4..d42f4db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,8 @@ requires-python = ">=3.10" authors = [{ name = "Rabii Chaarani" }] dependencies = [ "tomli; python_version < '3.11'", + "tree-sitter", + "tree-sitter-python", ] classifiers = [ "Programming Language :: Python :: 3", @@ -25,6 +27,9 @@ ladybug = ["real_ladybug"] parquet = ["pyarrow"] dev = ["pytest", "ruff"] +[project.scripts] +codebase-graph = "cli:main" + [tool.setuptools.packages.find] where = ["src"] diff --git a/src/cli/__init__.py b/src/cli/__init__.py index 9a82187..8167ca2 100644 --- a/src/cli/__init__.py +++ b/src/cli/__init__.py @@ -1 +1,53 @@ -"""Command-line entry points.""" +from __future__ import annotations + +import argparse +import json +from collections.abc import Sequence +from pathlib import Path + +from ingest import GraphMaterializer + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="codebase-graph") + subparsers = parser.add_subparsers(dest="command", required=True) + + materialize_parser = subparsers.add_parser("materialize", help="Materialize the code graph") + materialize_parser.add_argument("--source-root", default=".", help="Repository or source root to scan") + materialize_parser.add_argument("--db", default=None, help="LadybugDB path; defaults under .codebase_graph") + materialize_parser.add_argument("--manifest", default=None, help="Manifest path; defaults under .codebase_graph") + materialize_parser.add_argument("--mode", choices=("full", "changed"), default="changed") + materialize_parser.add_argument("--no-fts", action="store_true", help="Skip FTS index creation") + + args = parser.parse_args(argv) + if args.command == "materialize": + materializer = GraphMaterializer( + Path(args.source_root), + db_path=args.db, + manifest_path=args.manifest, + include_fts=not args.no_fts, + ) + result = materializer.materialize(mode=args.mode) + print(json.dumps(_result_payload(result), indent=2, sort_keys=True)) + return 0 + parser.error(f"Unknown command: {args.command}") + return 2 + + +def _result_payload(result: object) -> dict[str, object]: + return { + "mode": getattr(result, "mode"), + "scanned": getattr(result, "scanned"), + "rebuilt": getattr(result, "rebuilt"), + "skipped": getattr(result, "skipped"), + "deleted": getattr(result, "deleted"), + "diagnostics": list(getattr(result, "diagnostics")), + "manifest_path": getattr(result, "manifest_path"), + "rebuilt_paths": list(getattr(result, "rebuilt_paths")), + "skipped_paths": list(getattr(result, "skipped_paths")), + "deleted_paths": list(getattr(result, "deleted_paths")), + "graph_summary": dict(getattr(result, "graph_summary")), + } + + +__all__ = ["main"] diff --git a/src/db/__init__.py b/src/db/__init__.py index 57be53b..6213bea 100644 --- a/src/db/__init__.py +++ b/src/db/__init__.py @@ -1 +1,14 @@ """Storage adapters, schema management, and migrations.""" + +from .schema import build_ladybug_schema, build_ladybug_schema_statements, ladybug_type, quote_identifier +from .store import LadybugCodeGraphStore, LadybugUnavailableError, create_ladybug_database + +__all__ = [ + "LadybugCodeGraphStore", + "LadybugUnavailableError", + "build_ladybug_schema", + "build_ladybug_schema_statements", + "create_ladybug_database", + "ladybug_type", + "quote_identifier", +] diff --git a/src/db/schema.py b/src/db/schema.py new file mode 100644 index 0000000..9665d99 --- /dev/null +++ b/src/db/schema.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +from collections.abc import Iterable + +from ontology import EDGE_FIELDS, NODE_TYPES, RELATION_TYPES, SEARCH_INDEXES, FieldSpec + +TYPE_MAP = { + "string": "STRING", + "integer": "INT64", + "number": "DOUBLE", + "boolean": "BOOLEAN", + "json": "JSON", +} + + +def quote_identifier(name: str) -> str: + return f"`{name.replace('`', '``')}`" + + +def ladybug_type(value_type: str) -> str: + try: + return TYPE_MAP[value_type] + except KeyError as exc: + raise ValueError(f"Unsupported ontology field type for LadyBugDB: {value_type}") from exc + + +def build_ladybug_schema(*, include_fts: bool = True) -> str: + return ";\n\n".join(build_ladybug_schema_statements(include_fts=include_fts)) + ";" + + +def build_ladybug_schema_statements(*, include_fts: bool = True) -> list[str]: + statements = [ + "INSTALL json", + "LOAD json", + ] + if include_fts: + statements.extend(("INSTALL fts", "LOAD fts")) + statements.extend(_semantic_node_table_sql()) + statements.extend(_edge_node_table_sql()) + statements.extend(_connector_table_sql()) + if include_fts: + statements.extend(_fts_index_sql()) + return statements + + +def _semantic_node_table_sql() -> list[str]: + return [ + _node_table_sql(node_type.name, node_type.fields) + for node_type in NODE_TYPES + ] + + +def _edge_node_table_sql() -> list[str]: + return [ + _node_table_sql(relation_type.name, relation_type.fields or EDGE_FIELDS) + for relation_type in RELATION_TYPES + ] + + +def _connector_table_sql() -> list[str]: + statements: list[str] = [] + for relation_type in RELATION_TYPES: + relation_name = relation_type.name + source_pairs = _dedupe_pairs((source_type, relation_name) for source_type in relation_type.source_types) + target_pairs = _dedupe_pairs((relation_name, target_type) for target_type in relation_type.target_types) + statements.append(_relation_table_sql(f"FROM_{relation_name}", source_pairs, role="source")) + statements.append(_relation_table_sql(f"TO_{relation_name}", target_pairs, role="target")) + return statements + + +def _node_table_sql(table_name: str, fields: Iterable[FieldSpec]) -> str: + columns = [_field_sql(field) for field in _dedupe_fields(fields)] + return f"CREATE NODE TABLE IF NOT EXISTS {quote_identifier(table_name)}(\n" + ",\n".join(columns) + "\n)" + + +def _relation_table_sql(table_name: str, endpoint_pairs: Iterable[tuple[str, str]], *, role: str) -> str: + endpoints = [ + f" FROM {quote_identifier(source_type)} TO {quote_identifier(target_type)}" + for source_type, target_type in endpoint_pairs + ] + columns = [*endpoints, f" {quote_identifier('role')} STRING DEFAULT '{role}'"] + return f"CREATE REL TABLE IF NOT EXISTS {quote_identifier(table_name)}(\n" + ",\n".join(columns) + "\n)" + + +def _field_sql(field: FieldSpec) -> str: + primary_key = " PRIMARY KEY" if field.name == "id" else "" + return f" {quote_identifier(field.name)} {ladybug_type(field.value_type)}{primary_key}" + + +def _dedupe_fields(fields: Iterable[FieldSpec]) -> list[FieldSpec]: + seen: set[str] = set() + deduped: list[FieldSpec] = [] + for field in fields: + if field.name in seen: + continue + seen.add(field.name) + deduped.append(field) + return deduped + + +def _dedupe_pairs(pairs: Iterable[tuple[str, str]]) -> list[tuple[str, str]]: + seen: set[tuple[str, str]] = set() + deduped: list[tuple[str, str]] = [] + for pair in pairs: + if pair in seen: + continue + seen.add(pair) + deduped.append(pair) + return deduped + + +def _fts_index_sql() -> list[str]: + statements: list[str] = [] + for index in SEARCH_INDEXES: + fields = ", ".join(repr(field) for field in index["fields"]) + for node_type in index["node_types"]: + index_name = f"{index['name']}_{node_type}" + statements.append(f"CALL CREATE_FTS_INDEX('{node_type}', '{index_name}', [{fields}])") + return statements diff --git a/src/db/store.py b/src/db/store.py new file mode 100644 index 0000000..3cdd577 --- /dev/null +++ b/src/db/store.py @@ -0,0 +1,256 @@ +from __future__ import annotations + +import json +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +from core import CodeGraph, GraphEdge, GraphNode +from ontology import NODE_TYPES, RELATION_TYPES + +from .schema import build_ladybug_schema, build_ladybug_schema_statements, quote_identifier + + +class LadybugUnavailableError(RuntimeError): + pass + + +class LadybugCodeGraphStore: + def __init__(self, db_path: str | Path = ":memory:", *, include_fts: bool = True) -> None: + self.db_path = db_path + self.include_fts = include_fts + try: + import real_ladybug as lb + except ImportError as exc: + raise LadybugUnavailableError( + "LadyBugDB Python bindings are not installed. Install `real_ladybug` or `codebase-graph[ladybug]`." + ) from exc + + self._lb = lb + if str(db_path) != ":memory:": + Path(db_path).parent.mkdir(parents=True, exist_ok=True) + self.db = lb.Database(str(db_path)) + self.conn = lb.Connection(self.db) + + @property + def schema_sql(self) -> str: + return build_ladybug_schema(include_fts=self.include_fts) + + def ensure_schema(self) -> None: + for statement in build_ladybug_schema_statements(include_fts=self.include_fts): + self._execute_ignoring_existing(statement) + + def execute(self, statement: str, parameters: dict[str, Any] | None = None) -> Any: + if parameters is None: + return self.conn.execute(statement) + return self.conn.execute(statement, parameters) + + def clear_graph(self) -> None: + for relation_type in RELATION_TYPES: + self._execute_ignoring_missing(f"MATCH ()-[r:{quote_identifier(f'FROM_{relation_type.name}')}]->() DELETE r") + self._execute_ignoring_missing(f"MATCH ()-[r:{quote_identifier(f'TO_{relation_type.name}')}]->() DELETE r") + for relation_type in RELATION_TYPES: + self._execute_ignoring_missing(f"MATCH (n:{quote_identifier(relation_type.name)}) DELETE n") + for node_type in NODE_TYPES: + self._execute_ignoring_missing(f"MATCH (n:{quote_identifier(node_type.name)}) DELETE n") + + def replace_partition( + self, + path: str, + graph: CodeGraph, + *, + previous_entry: Mapping[str, Any] | Any | None = None, + retained_node_ids: set[str] | None = None, + ) -> None: + if previous_entry is not None: + self.delete_partition(path, manifest_entry=previous_entry, retained_node_ids=retained_node_ids) + + for node in graph.nodes.values(): + self._upsert_node(node) + for edge in graph.edges.values(): + self._upsert_edge_node(edge) + for edge in graph.edges.values(): + source = graph.nodes[edge.source_id] + target = graph.nodes[edge.target_id] + self._upsert_connector(edge, source, target) + + def delete_partition( + self, + path: str, + *, + manifest_entry: Mapping[str, Any] | Any | None = None, + retained_node_ids: set[str] | None = None, + ) -> None: + if manifest_entry is None: + return + retained = retained_node_ids or set() + edge_types = _entry_mapping(manifest_entry, "edge_types") + node_types = _entry_mapping(manifest_entry, "node_types") + + for edge_id in _entry_values(manifest_entry, "edge_ids"): + edge_type = edge_types.get(edge_id) + if edge_type: + self._delete_edge(edge_id, edge_type) + + for node_id in _entry_values(manifest_entry, "node_ids"): + if node_id in retained: + continue + node_type = node_types.get(node_id) + if node_type: + self._delete_node(node_id, node_type) + + def read_manifest(self, path: str | Path) -> Any: + from ingest.materializer import MaterializationManifest + + return MaterializationManifest.load(Path(path)) + + def write_manifest(self, manifest: Any, path: str | Path) -> None: + manifest.write(Path(path)) + + def _execute_ignoring_existing(self, statement: str) -> None: + try: + self.conn.execute(statement) + except Exception as exc: + message = str(exc).lower() + if "already exists" not in message and "exists already" not in message and "already installed" not in message: + raise + + def _execute_ignoring_missing(self, statement: str, parameters: dict[str, Any] | None = None) -> None: + try: + self.execute(statement, parameters) + except Exception as exc: + message = str(exc).lower() + if "does not exist" not in message and "not found" not in message: + raise + + def _upsert_node(self, node: GraphNode) -> None: + table_fields = NODE_FIELDS[node.table] + row = node.as_dict() + statement, parameters = _merge_statement(node.table, table_fields, row) + self.execute(statement, parameters) + + def _upsert_edge_node(self, edge: GraphEdge) -> None: + table_fields = EDGE_FIELDS_BY_TYPE[edge.type] + row = edge.as_dict() + statement, parameters = _merge_statement(edge.type, table_fields, row) + self.execute(statement, parameters) + + def _upsert_connector(self, edge: GraphEdge, source: GraphNode, target: GraphNode) -> None: + from_relation = quote_identifier(f"FROM_{edge.type}") + to_relation = quote_identifier(f"TO_{edge.type}") + self.execute( + ( + f"MATCH (source:{quote_identifier(source.table)} {{id: $source_id}}), " + f"(edge:{quote_identifier(edge.type)} {{id: $edge_id}}) " + f"MERGE (source)-[:{from_relation}]->(edge)" + ), + {"source_id": source.id, "edge_id": edge.id}, + ) + self.execute( + ( + f"MATCH (edge:{quote_identifier(edge.type)} {{id: $edge_id}}), " + f"(target:{quote_identifier(target.table)} {{id: $target_id}}) " + f"MERGE (edge)-[:{to_relation}]->(target)" + ), + {"edge_id": edge.id, "target_id": target.id}, + ) + + def _delete_edge(self, edge_id: str, edge_type: str) -> None: + self._execute_ignoring_missing( + f"MATCH ()-[r:{quote_identifier(f'FROM_{edge_type}')}]->(edge:{quote_identifier(edge_type)} {{id: $id}}) DELETE r", + {"id": edge_id}, + ) + self._execute_ignoring_missing( + f"MATCH (edge:{quote_identifier(edge_type)} {{id: $id}})-[r:{quote_identifier(f'TO_{edge_type}')}]->() DELETE r", + {"id": edge_id}, + ) + self._execute_ignoring_missing( + f"MATCH (edge:{quote_identifier(edge_type)} {{id: $id}}) DELETE edge", + {"id": edge_id}, + ) + + def _delete_node(self, node_id: str, node_type: str) -> None: + self._execute_ignoring_missing( + f"MATCH (node:{quote_identifier(node_type)} {{id: $id}}) DELETE node", + {"id": node_id}, + ) + + +def create_ladybug_database(db_path: str | Path = ":memory:", *, include_fts: bool = True) -> LadybugCodeGraphStore: + store = LadybugCodeGraphStore(db_path, include_fts=include_fts) + store.ensure_schema() + return store + + +NODE_FIELDS = { + node_type.name: tuple(field for field in node_type.fields) + for node_type in NODE_TYPES +} +_OMIT_JSON_VALUE = object() +EDGE_FIELDS_BY_TYPE = { + relation_type.name: tuple(field for field in relation_type.fields) + for relation_type in RELATION_TYPES +} + + +def _merge_statement(table: str, fields: tuple[Any, ...], row: Mapping[str, Any]) -> tuple[str, dict[str, Any]]: + parameters: dict[str, Any] = {"id": _field_value("id", row, "string")} + assignments = [] + for field in fields: + if field.name == "id": + continue + field_value = _field_value(field.name, row, field.value_type) + if field_value is None: + continue + parameters[field.name] = field_value + value = f"CAST(${field.name} AS JSON)" if field.value_type == "json" else f"${field.name}" + assignments.append(f"n.{quote_identifier(field.name)} = {value}") + statement = f"MERGE (n:{quote_identifier(table)} {{id: $id}})" + if assignments: + statement += f" SET {', '.join(assignments)}" + return statement, parameters + + +def _field_value(name: str, row: Mapping[str, Any], value_type: str) -> Any: + if name in row: + value = row[name] + else: + metadata = row.get("metadata") if isinstance(row.get("metadata"), Mapping) else {} + value = metadata.get(name) + if value_type == "json": + return json.dumps(_json_safe(value if value is not None else {}), sort_keys=True) + return value + + +def _json_safe(value: Any) -> Any: + if isinstance(value, Mapping): + safe_items = {} + for key, item in value.items(): + safe_item = _json_safe(item) + if safe_item is _OMIT_JSON_VALUE: + continue + safe_items[str(key)] = safe_item + return safe_items + if isinstance(value, list | tuple): + if not value: + return _OMIT_JSON_VALUE + return [_json_safe(item) for item in value] + if value is None: + return _OMIT_JSON_VALUE + return value + + +def _entry_values(entry: Mapping[str, Any] | Any, field_name: str) -> tuple[str, ...]: + if isinstance(entry, Mapping): + values = entry.get(field_name, ()) + else: + values = getattr(entry, field_name, ()) + return tuple(str(value) for value in values) + + +def _entry_mapping(entry: Mapping[str, Any] | Any, field_name: str) -> dict[str, str]: + if isinstance(entry, Mapping): + values = entry.get(field_name, {}) + else: + values = getattr(entry, field_name, {}) + return {str(key): str(value) for key, value in dict(values).items()} diff --git a/src/ingest/__init__.py b/src/ingest/__init__.py index 4e29ce6..64b2fad 100644 --- a/src/ingest/__init__.py +++ b/src/ingest/__init__.py @@ -1 +1,25 @@ """Repository, documentation, issue, and tool-output ingestion.""" + +from .materializer import ( + GraphMaterializer, + ManifestDiff, + ManifestEntry, + MaterializationManifest, + MaterializationResult, + MaterializeMode, + SourceSnapshot, +) +from .tree_sitter_parser import ParserUnavailableError, TreeSitterPythonParser, parser_for_language + +__all__ = [ + "GraphMaterializer", + "ManifestDiff", + "ManifestEntry", + "MaterializationManifest", + "MaterializationResult", + "MaterializeMode", + "ParserUnavailableError", + "SourceSnapshot", + "TreeSitterPythonParser", + "parser_for_language", +] diff --git a/src/ingest/materializer.py b/src/ingest/materializer.py new file mode 100644 index 0000000..2cc3dfd --- /dev/null +++ b/src/ingest/materializer.py @@ -0,0 +1,374 @@ +from __future__ import annotations + +import hashlib +import json +import os +from collections.abc import Mapping +from dataclasses import dataclass, field +from datetime import UTC, datetime +from pathlib import Path +from typing import Any, Literal + +from core import CodeGraph +from db import LadybugCodeGraphStore, create_ladybug_database +from extract import GraphBuilder +from ontology import ONTOLOGY_NAME + +from .tree_sitter_parser import ParserUnavailableError, parser_for_language + +MaterializeMode = Literal["full", "changed"] + +MANIFEST_SCHEMA_VERSION = 1 +DEFAULT_STATE_DIR = ".codebase_graph" +DEFAULT_MANIFEST_NAME = "manifest.json" +DEFAULT_DB_NAME = "graph.lbug" +PARSER_VERSION = "tree-sitter-python-v1" +SUPPORTED_SUFFIXES = {".py": "python"} +EXCLUDED_PARTS = { + ".git", + ".venv", + "__pycache__", + ".pytest_cache", + ".ruff_cache", + "build", + "dist", + ".codebase_graph", +} + + +@dataclass(frozen=True, slots=True) +class SourceSnapshot: + path: str + absolute_path: Path + content_hash: str + language: str | None + + +@dataclass(frozen=True, slots=True) +class ManifestEntry: + path: str + content_hash: str + language: str + partition_id: str + node_ids: tuple[str, ...] + edge_ids: tuple[str, ...] + node_types: Mapping[str, str] = field(default_factory=dict) + edge_types: Mapping[str, str] = field(default_factory=dict) + materialized_at: str = "" + + @classmethod + def from_dict(cls, payload: Mapping[str, Any]) -> ManifestEntry: + return cls( + path=str(payload["path"]), + content_hash=str(payload["content_hash"]), + language=str(payload["language"]), + partition_id=str(payload["partition_id"]), + node_ids=tuple(str(value) for value in payload.get("node_ids", ())), + edge_ids=tuple(str(value) for value in payload.get("edge_ids", ())), + node_types={str(key): str(value) for key, value in dict(payload.get("node_types", {})).items()}, + edge_types={str(key): str(value) for key, value in dict(payload.get("edge_types", {})).items()}, + materialized_at=str(payload.get("materialized_at", "")), + ) + + def as_dict(self) -> dict[str, Any]: + return { + "path": self.path, + "content_hash": self.content_hash, + "language": self.language, + "partition_id": self.partition_id, + "node_ids": list(self.node_ids), + "edge_ids": list(self.edge_ids), + "node_types": dict(self.node_types), + "edge_types": dict(self.edge_types), + "materialized_at": self.materialized_at, + } + + +@dataclass(frozen=True, slots=True) +class MaterializationManifest: + schema_version: int = MANIFEST_SCHEMA_VERSION + ontology: str = ONTOLOGY_NAME + parser_version: str = PARSER_VERSION + files: Mapping[str, ManifestEntry] = field(default_factory=dict) + + @classmethod + def empty(cls) -> MaterializationManifest: + return cls(files={}) + + @classmethod + def load(cls, path: Path) -> MaterializationManifest: + if not path.exists(): + return cls.empty() + with path.open("r", encoding="utf-8") as handle: + payload = json.load(handle) + return cls( + schema_version=int(payload.get("schema_version", 0)), + ontology=str(payload.get("ontology", "")), + parser_version=str(payload.get("parser_version", "")), + files={ + str(file_payload["path"]): ManifestEntry.from_dict(file_payload) + for file_payload in payload.get("files", []) + }, + ) + + def as_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "ontology": self.ontology, + "parser_version": self.parser_version, + "files": [entry.as_dict() for entry in sorted(self.files.values(), key=lambda item: item.path)], + } + + def write(self, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + tmp_path = path.with_suffix(path.suffix + ".tmp") + with tmp_path.open("w", encoding="utf-8") as handle: + json.dump(self.as_dict(), handle, indent=2, sort_keys=True) + handle.write("\n") + os.replace(tmp_path, path) + + def is_compatible(self) -> bool: + return ( + self.schema_version == MANIFEST_SCHEMA_VERSION + and self.ontology == ONTOLOGY_NAME + and self.parser_version == PARSER_VERSION + ) + + def diff(self, current_files: Mapping[str, SourceSnapshot]) -> ManifestDiff: + if not self.is_compatible(): + return ManifestDiff( + added=tuple(sorted(current_files)), + modified=(), + unchanged=(), + deleted=tuple(sorted(path for path in self.files if path not in current_files)), + force_rebuild=True, + ) + + added: list[str] = [] + modified: list[str] = [] + unchanged: list[str] = [] + for path, snapshot in current_files.items(): + previous = self.files.get(path) + if previous is None: + added.append(path) + elif previous.content_hash != snapshot.content_hash or previous.language != snapshot.language: + modified.append(path) + else: + unchanged.append(path) + + deleted = [path for path in self.files if path not in current_files] + return ManifestDiff( + added=tuple(sorted(added)), + modified=tuple(sorted(modified)), + unchanged=tuple(sorted(unchanged)), + deleted=tuple(sorted(deleted)), + force_rebuild=False, + ) + + +@dataclass(frozen=True, slots=True) +class ManifestDiff: + added: tuple[str, ...] + modified: tuple[str, ...] + unchanged: tuple[str, ...] + deleted: tuple[str, ...] + force_rebuild: bool = False + + @property + def rebuild_paths(self) -> tuple[str, ...]: + return tuple(sorted((*self.added, *self.modified))) + + +@dataclass(frozen=True, slots=True) +class MaterializationResult: + mode: MaterializeMode + scanned: int + rebuilt: int + skipped: int + deleted: int + diagnostics: tuple[str, ...] + manifest_path: str + rebuilt_paths: tuple[str, ...] + skipped_paths: tuple[str, ...] + deleted_paths: tuple[str, ...] + graph_summary: Mapping[str, Any] + + +class GraphMaterializer: + def __init__( + self, + source_root: str | Path, + db_path: str | Path | None = None, + *, + manifest_path: str | Path | None = None, + include_fts: bool = True, + repository_label: str | None = None, + store: LadybugCodeGraphStore | None = None, + ) -> None: + self.source_root = Path(source_root).resolve() + self.state_dir = self.source_root / DEFAULT_STATE_DIR + self.db_path = db_path if db_path is not None else self.state_dir / DEFAULT_DB_NAME + self.manifest_path = Path(manifest_path) if manifest_path is not None else self.state_dir / DEFAULT_MANIFEST_NAME + self.include_fts = include_fts + self.repository_label = repository_label or self.source_root.name or "repository" + self.store = store or create_ladybug_database(self.db_path, include_fts=include_fts) + self.builder = GraphBuilder(repository_label=self.repository_label, source_root=self.source_root) + + def materialize(self, mode: MaterializeMode = "changed") -> MaterializationResult: + if mode not in {"full", "changed"}: + raise ValueError(f"Unsupported materialization mode: {mode}") + + previous_manifest = self.store.read_manifest(self.manifest_path) + snapshots, diagnostics = self._scan_source_files() + supported = {path: snapshot for path, snapshot in snapshots.items() if snapshot.language is not None} + + if mode == "full": + diff = ManifestDiff( + added=tuple(sorted(supported)), + modified=(), + unchanged=(), + deleted=tuple(sorted(previous_manifest.files)), + force_rebuild=True, + ) + self.store.clear_graph() + retained_node_ids: set[str] = set() + else: + diff = previous_manifest.diff(supported) + if diff.force_rebuild: + self.store.clear_graph() + retained_node_ids = set() + else: + retained_node_ids = _retained_node_ids(previous_manifest, set(diff.rebuild_paths) | set(diff.deleted)) + for path in diff.deleted: + self.store.delete_partition( + path, + manifest_entry=previous_manifest.files.get(path), + retained_node_ids=retained_node_ids, + ) + + rebuilt_entries: dict[str, ManifestEntry] = {} + for path in diff.rebuild_paths: + snapshot = supported[path] + previous_entry = None if diff.force_rebuild else previous_manifest.files.get(path) + graph = self._build_graph(snapshot) + self.store.replace_partition( + path, + graph, + previous_entry=previous_entry, + retained_node_ids=retained_node_ids, + ) + rebuilt_entries[path] = _manifest_entry(snapshot, graph) + + next_files = { + path: entry + for path, entry in previous_manifest.files.items() + if path not in set(diff.deleted) | set(diff.rebuild_paths) + } + next_files.update(rebuilt_entries) + next_manifest = MaterializationManifest(files=next_files) + self.store.write_manifest(next_manifest, self.manifest_path) + + unsupported_paths = tuple(path for path, snapshot in snapshots.items() if snapshot.language is None) + skipped_paths = tuple(sorted((*diff.unchanged, *unsupported_paths))) + return MaterializationResult( + mode=mode, + scanned=len(snapshots), + rebuilt=len(rebuilt_entries), + skipped=len(skipped_paths), + deleted=len(diff.deleted), + diagnostics=tuple(diagnostics), + manifest_path=self.manifest_path.as_posix(), + rebuilt_paths=tuple(sorted(rebuilt_entries)), + skipped_paths=skipped_paths, + deleted_paths=diff.deleted, + graph_summary=_manifest_summary(next_manifest), + ) + + def _scan_source_files(self) -> tuple[dict[str, SourceSnapshot], list[str]]: + snapshots: dict[str, SourceSnapshot] = {} + diagnostics: list[str] = [] + for path in sorted(self.source_root.rglob("*")): + if not path.is_file() or _is_excluded(path, self.source_root): + continue + relative_path = path.relative_to(self.source_root).as_posix() + language = SUPPORTED_SUFFIXES.get(path.suffix) + snapshots[relative_path] = SourceSnapshot( + path=relative_path, + absolute_path=path, + content_hash=_file_hash(path), + language=language, + ) + if language is None: + diagnostics.append(f"Skipped unsupported file: {relative_path}") + return snapshots, diagnostics + + def _build_graph(self, snapshot: SourceSnapshot) -> CodeGraph: + if snapshot.language is None: + raise ValueError(f"Cannot build graph for unsupported file: {snapshot.path}") + try: + parser = parser_for_language(snapshot.language) + bundle = parser.parse_file( + snapshot.absolute_path, + relative_path=snapshot.path, + source_root=self.source_root, + repository_label=self.repository_label, + content_hash=snapshot.content_hash, + ) + except ParserUnavailableError: + raise + result = self.builder.build_file_graph(bundle) + return result.graph + + +def _is_excluded(path: Path, source_root: Path) -> bool: + parts = path.relative_to(source_root).parts + return any(part in EXCLUDED_PARTS or part.endswith(".egg-info") for part in parts) + + +def _file_hash(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _partition_id(path: str) -> str: + return hashlib.sha1(path.encode("utf-8")).hexdigest()[:20] + + +def _manifest_entry(snapshot: SourceSnapshot, graph: CodeGraph) -> ManifestEntry: + return ManifestEntry( + path=snapshot.path, + content_hash=snapshot.content_hash, + language=snapshot.language or "", + partition_id=_partition_id(snapshot.path), + node_ids=tuple(sorted(graph.nodes)), + edge_ids=tuple(sorted(graph.edges)), + node_types={node_id: node.table for node_id, node in graph.nodes.items()}, + edge_types={edge_id: edge.type for edge_id, edge in graph.edges.items()}, + materialized_at=datetime.now(UTC).isoformat(), + ) + + +def _retained_node_ids(manifest: MaterializationManifest, touched_paths: set[str]) -> set[str]: + retained: set[str] = set() + for path, entry in manifest.files.items(): + if path in touched_paths: + continue + retained.update(entry.node_ids) + return retained + + +def _manifest_summary(manifest: MaterializationManifest) -> dict[str, int | str]: + node_ids: set[str] = set() + edge_ids: set[str] = set() + for entry in manifest.files.values(): + node_ids.update(entry.node_ids) + edge_ids.update(entry.edge_ids) + return { + "ontology": manifest.ontology, + "partition_count": len(manifest.files), + "node_count": len(node_ids), + "edge_count": len(edge_ids), + } diff --git a/src/ingest/tree_sitter_parser.py b/src/ingest/tree_sitter_parser.py new file mode 100644 index 0000000..21f307a --- /dev/null +++ b/src/ingest/tree_sitter_parser.py @@ -0,0 +1,250 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from extract import ParseBundle + + +class ParserUnavailableError(RuntimeError): + pass + + +@dataclass(frozen=True, slots=True) +class TreeSitterPythonParser: + language: str = "python" + parser_version: str = "tree-sitter-python-v1" + + def parse_file( + self, + path: Path, + *, + relative_path: str, + source_root: Path, + repository_label: str, + content_hash: str, + ) -> ParseBundle: + source_text = path.read_text(encoding="utf-8") + return ParseBundle( + language=self.language, + path=relative_path, + source_text=source_text, + tree=self.parse_source(source_text), + repository_label=repository_label, + source_root=source_root.as_posix(), + content_hash=content_hash, + ) + + def parse_source(self, source_text: str) -> dict[str, Any]: + parser = _python_parser() + source_bytes = source_text.encode("utf-8") + tree = parser.parse(source_bytes) + return _convert_node(tree.root_node, source_bytes) + + +def parser_for_language(language: str) -> TreeSitterPythonParser: + if language == "python": + return TreeSitterPythonParser() + raise ValueError(f"Unsupported materializer language: {language}") + + +def _python_parser() -> Any: + try: + from tree_sitter import Language, Parser + import tree_sitter_python + except ImportError as exc: + raise ParserUnavailableError( + "Tree-sitter Python parsing requires `tree-sitter` and `tree-sitter-python`." + ) from exc + + raw_language = tree_sitter_python.language() + try: + language = Language(raw_language) + except TypeError: + language = raw_language + + parser = Parser() + if hasattr(parser, "set_language"): + parser.set_language(language) + else: + parser.language = language + return parser + + +def _convert_node(node: Any, source_bytes: bytes, decorators: tuple[dict[str, Any], ...] = ()) -> dict[str, Any]: + if node.type == "decorated_definition": + converted_decorators = tuple( + _convert_node(child, source_bytes) + for child in _named_children(node) + if child.type == "decorator" + ) + for child in _named_children(node): + if child.type in {"class_definition", "function_definition"}: + return _convert_node(child, source_bytes, converted_decorators) + + converted: dict[str, Any] = { + "type": node.type, + "text": _node_text(node, source_bytes), + "line_start": _line_start(node), + "line_end": _line_end(node), + "byte_start": node.start_byte, + "byte_end": node.end_byte, + } + + if node.type == "module": + converted["children"] = [_convert_node(child, source_bytes) for child in _named_children(node)] + elif node.type == "class_definition": + converted.update(_class_fields(node, source_bytes, decorators)) + elif node.type == "function_definition": + converted.update(_function_fields(node, source_bytes, decorators)) + elif node.type in {"import_statement", "import_from_statement"}: + converted.update(_import_fields(node, source_bytes)) + elif node.type == "call": + converted.update(_call_fields(node, source_bytes)) + elif node.type == "assignment": + converted.update(_assignment_fields(node, source_bytes)) + elif node.type in {"identifier", "type_identifier"}: + converted["id"] = _node_text(node, source_bytes) + elif node.type == "attribute": + converted.update(_attribute_fields(node, source_bytes)) + elif node.type in {"string", "integer", "float", "true", "false", "none"}: + converted["value"] = _literal_value(node, source_bytes) + + converted.setdefault("children", [_convert_node(child, source_bytes) for child in _semantic_children(node)]) + return converted + + +def _class_fields( + node: Any, + source_bytes: bytes, + decorators: tuple[dict[str, Any], ...], +) -> dict[str, Any]: + fields: dict[str, Any] = {"name": _field_text(node, "name", source_bytes)} + if decorators: + fields["decorator_list"] = list(decorators) + body = node.child_by_field_name("body") + fields["children"] = [_convert_node(child, source_bytes) for child in _named_children(body)] + return fields + + +def _function_fields( + node: Any, + source_bytes: bytes, + decorators: tuple[dict[str, Any], ...], +) -> dict[str, Any]: + fields: dict[str, Any] = {"name": _field_text(node, "name", source_bytes)} + parameters = node.child_by_field_name("parameters") + if parameters is not None: + fields["args"] = {"type": "arguments", "args": [_parameter_node(child, source_bytes) for child in _named_children(parameters)]} + return_type = node.child_by_field_name("return_type") + if return_type is not None: + fields["returns"] = _convert_node(return_type, source_bytes) + if decorators: + fields["decorator_list"] = list(decorators) + body = node.child_by_field_name("body") + fields["children"] = [_convert_node(child, source_bytes) for child in _named_children(body)] + return fields + + +def _parameter_node(node: Any, source_bytes: bytes) -> dict[str, Any]: + text = _node_text(node, source_bytes) + name = text.split(":", 1)[0].split("=", 1)[0].strip().lstrip("*") + parameter: dict[str, Any] = { + "type": "arg", + "arg": name, + "text": text, + "line_start": _line_start(node), + "line_end": _line_end(node), + "byte_start": node.start_byte, + "byte_end": node.end_byte, + } + if ":" in text: + annotation = text.split(":", 1)[1].split("=", 1)[0].strip() + if annotation: + parameter["annotation"] = {"type": "type", "id": annotation, "text": annotation} + return parameter + + +def _import_fields(node: Any, source_bytes: bytes) -> dict[str, Any]: + text = _node_text(node, source_bytes).strip() + if node.type == "import_from_statement": + match = re.match(r"from\s+([.\w]+)\s+import\s+(.+)", text) + if match: + names = [_import_alias(name) for name in match.group(2).split(",")] + return {"module": match.group(1), "names": names} + if node.type == "import_statement": + imported = text.removeprefix("import").strip() + return {"names": [_import_alias(name) for name in imported.split(",")]} + return {} + + +def _import_alias(raw_name: str) -> dict[str, str]: + name = raw_name.strip().split(" as ", 1)[0].strip() + return {"type": "alias", "name": name} + + +def _call_fields(node: Any, source_bytes: bytes) -> dict[str, Any]: + function = node.child_by_field_name("function") + if function is not None: + return {"func": _convert_node(function, source_bytes)} + text = _node_text(node, source_bytes) + return {"func": {"type": "identifier", "id": text.split("(", 1)[0].strip()}} + + +def _assignment_fields(node: Any, source_bytes: bytes) -> dict[str, Any]: + left = node.child_by_field_name("left") + right = node.child_by_field_name("right") + fields: dict[str, Any] = {} + if left is not None: + fields["target"] = _convert_node(left, source_bytes) + if right is not None: + fields["value"] = _convert_node(right, source_bytes) + return fields + + +def _attribute_fields(node: Any, source_bytes: bytes) -> dict[str, Any]: + text = _node_text(node, source_bytes) + if "." not in text: + return {"id": text} + base, attr = text.rsplit(".", 1) + return {"value": {"type": "identifier", "id": base}, "attr": attr} + + +def _literal_value(node: Any, source_bytes: bytes) -> str: + return _node_text(node, source_bytes).strip("'\"") + + +def _semantic_children(node: Any) -> tuple[Any, ...]: + ignored = {"identifier", "type_identifier", "parameters", "decorator", "block"} + return tuple(child for child in _named_children(node) if child.type not in ignored) + + +def _named_children(node: Any | None) -> tuple[Any, ...]: + if node is None: + return () + return tuple(getattr(node, "named_children", ()) or ()) + + +def _field_text(node: Any, field_name: str, source_bytes: bytes) -> str: + child = node.child_by_field_name(field_name) + return _node_text(child, source_bytes) if child is not None else "" + + +def _node_text(node: Any, source_bytes: bytes) -> str: + return source_bytes[node.start_byte:node.end_byte].decode("utf-8", errors="replace") + + +def _line_start(node: Any) -> int: + return _point_row(node.start_point) + 1 + + +def _line_end(node: Any) -> int: + return _point_row(node.end_point) + 1 + + +def _point_row(point: Any) -> int: + if hasattr(point, "row"): + return int(point.row) + return int(point[0]) diff --git a/src/tests/test_materializer.py b/src/tests/test_materializer.py new file mode 100644 index 0000000..73a61bd --- /dev/null +++ b/src/tests/test_materializer.py @@ -0,0 +1,151 @@ +from __future__ import annotations + +import shutil +from pathlib import Path + +import pytest + +from ingest import ( + GraphMaterializer, + ManifestEntry, + MaterializationManifest, + SourceSnapshot, + TreeSitterPythonParser, +) +from ontology import ONTOLOGY_NAME + + +def test_manifest_diff_tracks_added_modified_unchanged_and_deleted(tmp_path: Path) -> None: + manifest = MaterializationManifest( + files={ + "same.py": _entry("same.py", "same"), + "changed.py": _entry("changed.py", "old"), + "deleted.py": _entry("deleted.py", "old"), + } + ) + current = { + "same.py": _snapshot(tmp_path, "same.py", "same"), + "changed.py": _snapshot(tmp_path, "changed.py", "new"), + "added.py": _snapshot(tmp_path, "added.py", "new"), + } + + diff = manifest.diff(current) + + assert diff.added == ("added.py",) + assert diff.modified == ("changed.py",) + assert diff.unchanged == ("same.py",) + assert diff.deleted == ("deleted.py",) + assert diff.rebuild_paths == ("added.py", "changed.py") + assert not diff.force_rebuild + + +def test_manifest_diff_forces_rebuild_on_contract_mismatch(tmp_path: Path) -> None: + manifest = MaterializationManifest(schema_version=0, ontology=ONTOLOGY_NAME, parser_version="old", files={}) + current = {"service.py": _snapshot(tmp_path, "service.py", "hash")} + + diff = manifest.diff(current) + + assert diff.force_rebuild + assert diff.added == ("service.py",) + assert diff.rebuild_paths == ("service.py",) + + +def test_tree_sitter_python_parser_maps_sample_fixture_to_graph_tree() -> None: + pytest.importorskip("tree_sitter") + pytest.importorskip("tree_sitter_python") + fixture = Path("tests/fixtures/sample_project/sample_project/service.py") + parser = TreeSitterPythonParser() + + bundle = parser.parse_file( + fixture, + relative_path="sample_project/service.py", + source_root=fixture.parents[1], + repository_label="sample", + content_hash="hash", + ) + + assert bundle.language == "python" + assert bundle.tree["type"] == "module" + assert any(child["type"] == "class_definition" and child["name"] == "SampleService" for child in bundle.tree["children"]) + assert any(child["type"] == "function_definition" and child["name"] == "helper" for child in bundle.tree["children"]) + + +def test_full_materialization_writes_python_graph_to_ladybug(tmp_path: Path) -> None: + pytest.importorskip("tree_sitter") + pytest.importorskip("tree_sitter_python") + pytest.importorskip("real_ladybug") + source_root = _copy_fixture(tmp_path) + + materializer = GraphMaterializer( + source_root, + db_path=":memory:", + manifest_path=tmp_path / "manifest.json", + include_fts=False, + ) + result = materializer.materialize(mode="full") + + assert result.rebuilt == 3 + assert result.deleted == 0 + assert result.graph_summary["partition_count"] == 3 + assert _labels(materializer, "File") == { + "__init__.py", + "cli.py", + "service.py", + } + assert "SampleService" in _labels(materializer, "Class") + assert "run" in _labels(materializer, "Method") + assert {"helper", "main"} <= _labels(materializer, "Function") + + +def test_changed_materialization_only_rebuilds_changed_files(tmp_path: Path) -> None: + pytest.importorskip("tree_sitter") + pytest.importorskip("tree_sitter_python") + pytest.importorskip("real_ladybug") + source_root = _copy_fixture(tmp_path) + manifest_path = tmp_path / "manifest.json" + materializer = GraphMaterializer(source_root, db_path=":memory:", manifest_path=manifest_path, include_fts=False) + + first = materializer.materialize(mode="changed") + second = materializer.materialize(mode="changed") + service_path = source_root / "sample_project" / "service.py" + service_path.write_text(service_path.read_text(encoding="utf-8") + "\n\ndef added() -> str:\n return 'added'\n", encoding="utf-8") + third = materializer.materialize(mode="changed") + (source_root / "sample_project" / "cli.py").unlink() + fourth = materializer.materialize(mode="changed") + + assert first.rebuilt == 3 + assert second.rebuilt == 0 + assert third.rebuilt == 1 + assert third.rebuilt_paths == ("sample_project/service.py",) + assert "added" in _labels(materializer, "Function") + assert fourth.rebuilt == 0 + assert fourth.deleted == 1 + assert fourth.deleted_paths == ("sample_project/cli.py",) + assert "cli.py" not in _labels(materializer, "File") + + +def _entry(path: str, content_hash: str) -> ManifestEntry: + return ManifestEntry( + path=path, + content_hash=content_hash, + language="python", + partition_id=path, + node_ids=(), + edge_ids=(), + ) + + +def _snapshot(tmp_path: Path, path: str, content_hash: str) -> SourceSnapshot: + return SourceSnapshot(path=path, absolute_path=tmp_path / path, content_hash=content_hash, language="python") + + +def _copy_fixture(tmp_path: Path) -> Path: + source = Path("tests/fixtures/sample_project") + target = tmp_path / "sample_project" + shutil.copytree(source, target) + return target + + +def _labels(materializer: GraphMaterializer, table: str) -> set[str]: + result = materializer.store.execute(f"MATCH (n:`{table}`) RETURN n.label") + return {row[0] for row in result.get_all()} diff --git a/src/tests/test_schema.py b/src/tests/test_schema.py new file mode 100644 index 0000000..95eb88d --- /dev/null +++ b/src/tests/test_schema.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +import pytest + +from db import ( + LadybugCodeGraphStore, + build_ladybug_schema, + build_ladybug_schema_statements, + create_ladybug_database, + ladybug_type, + quote_identifier, +) +from ontology import NODE_TYPES, RELATION_TYPES, SEARCH_INDEXES + + +def test_ladybug_schema_declares_all_ontology_nodes_and_edge_nodes() -> None: + schema = build_ladybug_schema() + + for node_type in NODE_TYPES: + assert f"CREATE NODE TABLE IF NOT EXISTS `{node_type.name}`" in schema + for relation_type in RELATION_TYPES: + assert f"CREATE NODE TABLE IF NOT EXISTS `{relation_type.name}`" in schema + + +def test_ladybug_schema_declares_from_and_to_connector_tables() -> None: + schema = build_ladybug_schema() + + for relation_type in RELATION_TYPES: + assert f"CREATE REL TABLE IF NOT EXISTS `FROM_{relation_type.name}`" in schema + assert f"CREATE REL TABLE IF NOT EXISTS `TO_{relation_type.name}`" in schema + + for source_type in set(relation_type.source_types): + assert f"FROM `{source_type}` TO `{relation_type.name}`" in schema + for target_type in set(relation_type.target_types): + assert f"FROM `{relation_type.name}` TO `{target_type}`" in schema + + +def test_ladybug_schema_keeps_relation_payload_on_edge_nodes() -> None: + schema = build_ladybug_schema() + + contains_start = schema.index("CREATE NODE TABLE IF NOT EXISTS `Contains`") + from_contains_start = schema.index("CREATE REL TABLE IF NOT EXISTS `FROM_Contains`") + contains_table = schema[contains_start:from_contains_start] + + assert "`id` STRING PRIMARY KEY" in contains_table + assert "`kind` STRING" in contains_table + assert "`source_id` STRING" in contains_table + assert "`target_id` STRING" in contains_table + assert "`confidence` DOUBLE" in contains_table + assert "`metadata` JSON" in contains_table + + +def test_ladybug_schema_maps_types_and_quotes_identifiers() -> None: + assert ladybug_type("string") == "STRING" + assert ladybug_type("integer") == "INT64" + assert ladybug_type("number") == "DOUBLE" + assert ladybug_type("boolean") == "BOOLEAN" + assert ladybug_type("json") == "JSON" + assert quote_identifier("Query") == "`Query`" + assert quote_identifier("odd`name") == "`odd``name`" + + +def test_ladybug_schema_rejects_unknown_field_type() -> None: + with pytest.raises(ValueError, match="Unsupported ontology field type"): + ladybug_type("object") + + +def test_ladybug_schema_deduplicates_connector_endpoint_pairs() -> None: + schema = build_ladybug_schema() + + assert schema.count("FROM `Contains` TO `Assignment`") == 1 + assert schema.count("FROM `Contains` TO `Query`") == 1 + assert "TO `Assignment` |" not in schema + + +def test_ladybug_schema_creates_fts_indexes_for_semantic_node_tables_only() -> None: + statements = build_ladybug_schema_statements(include_fts=True) + schema = ";\n".join(statements) + relation_names = {relation_type.name for relation_type in RELATION_TYPES} + + assert "INSTALL fts" in statements + for index in SEARCH_INDEXES: + for node_type in index["node_types"]: + assert f"CALL CREATE_FTS_INDEX('{node_type}', '{index['name']}_{node_type}'" in schema + assert node_type not in relation_names + + +def test_ladybug_schema_can_skip_fts_statements() -> None: + statements = build_ladybug_schema_statements(include_fts=False) + + assert "INSTALL json" in statements + assert "LOAD json" in statements + assert "INSTALL fts" not in statements + assert "LOAD fts" not in statements + assert not any(statement.startswith("CALL CREATE_FTS_INDEX") for statement in statements) + + +def test_ladybug_schema_executes_against_in_memory_database() -> None: + real_ladybug = pytest.importorskip("real_ladybug") + conn = real_ladybug.Connection(real_ladybug.Database(":memory:")) + + for statement in build_ladybug_schema_statements(): + conn.execute(statement) + + +def test_ladybug_store_creates_in_memory_database_without_persistent_file() -> None: + pytest.importorskip("real_ladybug") + + store = create_ladybug_database(":memory:") + + assert isinstance(store, LadybugCodeGraphStore) + assert store.schema_sql.startswith("INSTALL json") + + +def test_ladybug_store_schema_setup_is_idempotent() -> None: + pytest.importorskip("real_ladybug") + store = create_ladybug_database(":memory:") + + store.ensure_schema() From 105f4ae541522c4421242c02f3880c218904a135 Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Fri, 22 May 2026 11:44:25 +0930 Subject: [PATCH 07/53] feat: Add search functionality --- src/cli/__init__.py | 44 +++++++ src/reasoning/__init__.py | 4 + src/reasoning/context_builder.py | 207 +++++++++++++++++++++++++++++++ src/retrieval/__init__.py | 4 + src/retrieval/search.py | 199 +++++++++++++++++++++++++++++ src/tests/test_search.py | 194 +++++++++++++++++++++++++++++ 6 files changed, 652 insertions(+) create mode 100644 src/reasoning/context_builder.py create mode 100644 src/retrieval/search.py create mode 100644 src/tests/test_search.py diff --git a/src/cli/__init__.py b/src/cli/__init__.py index 8167ca2..479b03d 100644 --- a/src/cli/__init__.py +++ b/src/cli/__init__.py @@ -6,6 +6,8 @@ from pathlib import Path from ingest import GraphMaterializer +from ontology import CONTEXT_PROFILES +from retrieval import SearchRequest, SearchService def main(argv: Sequence[str] | None = None) -> int: @@ -19,6 +21,12 @@ def main(argv: Sequence[str] | None = None) -> int: materialize_parser.add_argument("--mode", choices=("full", "changed"), default="changed") materialize_parser.add_argument("--no-fts", action="store_true", help="Skip FTS index creation") + search_parser = subparsers.add_parser("search", help="Search the code graph with compact context") + _add_search_arguments(search_parser) + + context_parser = subparsers.add_parser("context", help="Return compact context for a search query") + _add_search_arguments(context_parser) + args = parser.parse_args(argv) if args.command == "materialize": materializer = GraphMaterializer( @@ -30,10 +38,46 @@ def main(argv: Sequence[str] | None = None) -> int: result = materializer.materialize(mode=args.mode) print(json.dumps(_result_payload(result), indent=2, sort_keys=True)) return 0 + if args.command in {"search", "context"}: + request = SearchRequest( + query=args.query, + limit=args.limit, + profile=args.profile, + budget=args.budget, + max_depth=args.max_depth, + ) + try: + request.validate() + except ValueError as exc: + parser.error(str(exc)) + materializer = GraphMaterializer( + Path(args.source_root), + db_path=args.db, + manifest_path=args.manifest, + include_fts=True, + ) + if not args.no_refresh: + materializer.materialize(mode="changed") + payload = SearchService(materializer.store).search(request) + print(json.dumps(payload.as_dict(), indent=2, sort_keys=True)) + return 0 parser.error(f"Unknown command: {args.command}") return 2 +def _add_search_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument("query", help="Search query") + parser.add_argument("--source-root", default=".", help="Repository or source root to search") + parser.add_argument("--db", default=None, help="LadybugDB path; defaults under .codebase_graph") + parser.add_argument("--manifest", default=None, help="Manifest path; defaults under .codebase_graph") + parser.add_argument("--limit", type=int, default=3, help="Maximum search hits to return") + parser.add_argument("--profile", choices=sorted(CONTEXT_PROFILES), default="brief", help="Context profile") + parser.add_argument("--budget", type=int, default=600, help="Approximate per-hit context character budget") + parser.add_argument("--max-depth", type=int, default=None, help="Override the context profile depth") + parser.add_argument("--no-refresh", action="store_true", help="Query the existing graph without changed materialization") + parser.add_argument("--json", action="store_true", help="Emit compact JSON output") + + def _result_payload(result: object) -> dict[str, object]: return { "mode": getattr(result, "mode"), diff --git a/src/reasoning/__init__.py b/src/reasoning/__init__.py index d26618a..44b7855 100644 --- a/src/reasoning/__init__.py +++ b/src/reasoning/__init__.py @@ -1 +1,5 @@ """Path explanation, causal trace, and context assembly.""" + +from .context_builder import CompactContextBuilder, ContextNode + +__all__ = ["CompactContextBuilder", "ContextNode"] diff --git a/src/reasoning/context_builder.py b/src/reasoning/context_builder.py new file mode 100644 index 0000000..1f0b49a --- /dev/null +++ b/src/reasoning/context_builder.py @@ -0,0 +1,207 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from db.schema import quote_identifier +from ontology import CONTEXT_PROFILES, RELATION_TYPES + + +DEFAULT_CONTEXT_LIMIT = 3 +DEFAULT_CONTEXT_BUDGET = 600 + + +@dataclass(frozen=True, slots=True) +class ContextNode: + relation: str + direction: str + type: str + label: str + path: str = "" + span: dict[str, int] = field(default_factory=dict) + summary: str = "" + id: str = field(default="", repr=False) + + def as_dict(self) -> dict[str, Any]: + return { + "relation": self.relation, + "direction": self.direction, + "type": self.type, + "label": self.label, + "path": self.path, + "span": dict(self.span), + "summary": self.summary, + } + + +class CompactContextBuilder: + def __init__(self, store: Any) -> None: + self.store = store + self._relation_names = {relation_type.name for relation_type in RELATION_TYPES} + + def build( + self, + node_id: str, + node_type: str, + *, + profile: str = "brief", + limit: int = DEFAULT_CONTEXT_LIMIT, + budget: int = DEFAULT_CONTEXT_BUDGET, + max_depth: int | None = None, + ) -> list[ContextNode]: + profile_config = self._profile(profile) + if limit <= 0 or budget <= 0: + return [] + depth_limit = profile_config["max_depth"] if max_depth is None else max_depth + if depth_limit <= 0: + return [] + + relations = tuple( + relation + for relation in profile_config["relations"] + if relation in self._relation_names + ) + if not relations: + return [] + + context: list[ContextNode] = [] + seen = {node_id} + frontier = [(node_id, node_type, 0)] + used_budget = 0 + + while frontier and len(context) < limit: + current_id, current_type, depth = frontier.pop(0) + if depth >= depth_limit: + continue + for relation in relations: + for candidate in self._neighbors(current_id, current_type, relation, limit): + if candidate.type == "" or candidate.label == "": + continue + candidate_key = f"{candidate.type}:{candidate.label}:{candidate.path}:{candidate.span}" + node_key = _node_key(candidate) + dedupe_key = node_key or candidate_key + if dedupe_key in seen: + continue + compact_candidate, item_cost = _fit_to_budget(candidate, budget - used_budget) + if compact_candidate is None: + return context + context.append(compact_candidate) + used_budget += item_cost + seen.add(dedupe_key) + if node_key: + frontier.append((node_key, candidate.type, depth + 1)) + if len(context) >= limit: + return context + return context + + def _profile(self, profile: str) -> dict[str, Any]: + if profile not in CONTEXT_PROFILES: + valid = ", ".join(sorted(CONTEXT_PROFILES)) + raise ValueError(f"Unknown context profile: {profile}. Valid profiles: {valid}") + return dict(CONTEXT_PROFILES[profile]) + + def _neighbors(self, node_id: str, node_type: str, relation: str, limit: int) -> list[ContextNode]: + outgoing = self._query_neighbors(node_id, node_type, relation, "outgoing", limit) + incoming = self._query_neighbors(node_id, node_type, relation, "incoming", limit) + return [*outgoing, *incoming] + + def _query_neighbors( + self, + node_id: str, + node_type: str, + relation: str, + direction: str, + limit: int, + ) -> list[ContextNode]: + if direction == "outgoing": + statement = ( + f"MATCH (source:{quote_identifier(node_type)} {{id: $node_id}})" + f"-[:{quote_identifier(f'FROM_{relation}')}]->(edge:{quote_identifier(relation)})" + f"-[:{quote_identifier(f'TO_{relation}')}]->(neighbor) " + "RETURN neighbor.id, neighbor.label, neighbor.qualified_name, neighbor.path, " + f"neighbor.line_start, neighbor.line_end, neighbor.summary LIMIT {int(limit)}" + ) + else: + statement = ( + "MATCH (neighbor)" + f"-[:{quote_identifier(f'FROM_{relation}')}]->(edge:{quote_identifier(relation)})" + f"-[:{quote_identifier(f'TO_{relation}')}]->(target:{quote_identifier(node_type)} {{id: $node_id}}) " + "RETURN neighbor.id, neighbor.label, neighbor.qualified_name, neighbor.path, " + f"neighbor.line_start, neighbor.line_end, neighbor.summary LIMIT {int(limit)}" + ) + rows = self.store.execute(statement, {"node_id": node_id}).get_all() + return [ + ContextNode( + relation=relation, + direction=direction, + type=_type_from_id(_value(row, 0)), + label=_text(_value(row, 1)) or _text(_value(row, 2)), + path=_text(_value(row, 3)), + span=_span(_value(row, 4), _value(row, 5)), + summary=_text(_value(row, 6)), + id=_text(_value(row, 0)), + ) + for row in rows + ] + + +def _fit_to_budget(node: ContextNode, remaining_budget: int) -> tuple[ContextNode | None, int]: + cost = _context_cost(node) + if cost <= remaining_budget: + return node, cost + fixed_cost = _context_cost(ContextNode(node.relation, node.direction, node.type, node.label, node.path, node.span, "")) + summary_budget = remaining_budget - fixed_cost + if summary_budget <= 0: + return None, 0 + summary = node.summary[:summary_budget] + compact = ContextNode(node.relation, node.direction, node.type, node.label, node.path, node.span, summary) + return compact, _context_cost(compact) + + +def _context_cost(node: ContextNode) -> int: + return sum( + len(str(value)) + for value in ( + node.relation, + node.direction, + node.type, + node.label, + node.path, + node.summary, + *node.span.values(), + ) + ) + + +def _node_key(node: ContextNode) -> str: + return node.id + + +def _type_from_id(value: Any) -> str: + text = _text(value) + if ":" not in text: + return "" + return text.split(":", 1)[0] + + +def _span(line_start: Any, line_end: Any) -> dict[str, int]: + span: dict[str, int] = {} + if line_start is not None: + span["line_start"] = int(line_start) + if line_end is not None: + span["line_end"] = int(line_end) + return span + + +def _text(value: Any) -> str: + return "" if value is None else str(value) + + +def _value(row: Any, index: int) -> Any: + try: + return row[index] + except IndexError: + return None + + +__all__ = ["CompactContextBuilder", "ContextNode", "DEFAULT_CONTEXT_BUDGET", "DEFAULT_CONTEXT_LIMIT"] diff --git a/src/retrieval/__init__.py b/src/retrieval/__init__.py index 53a3413..cd58222 100644 --- a/src/retrieval/__init__.py +++ b/src/retrieval/__init__.py @@ -1 +1,5 @@ """Keyword, vector, graph traversal, and ranking retrieval.""" + +from .search import CompactContextPayload, SearchHit, SearchRequest, SearchService + +__all__ = ["CompactContextPayload", "SearchHit", "SearchRequest", "SearchService"] diff --git a/src/retrieval/search.py b/src/retrieval/search.py new file mode 100644 index 0000000..32d5ef6 --- /dev/null +++ b/src/retrieval/search.py @@ -0,0 +1,199 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from ontology import CONTEXT_PROFILES, SEARCH_INDEXES +from reasoning.context_builder import CompactContextBuilder, ContextNode, DEFAULT_CONTEXT_BUDGET, DEFAULT_CONTEXT_LIMIT + + +DEFAULT_SEARCH_LIMIT = 3 + + +@dataclass(frozen=True, slots=True) +class SearchRequest: + query: str + limit: int = DEFAULT_SEARCH_LIMIT + profile: str = "brief" + budget: int = DEFAULT_CONTEXT_BUDGET + max_depth: int | None = None + + def validate(self) -> None: + if not self.query.strip(): + raise ValueError("Search query must not be empty") + if self.limit <= 0: + raise ValueError("Search limit must be greater than zero") + if self.budget < 0: + raise ValueError("Context budget must be zero or greater") + if self.max_depth is not None and self.max_depth < 0: + raise ValueError("Context max depth must be zero or greater") + if self.profile not in CONTEXT_PROFILES: + valid = ", ".join(sorted(CONTEXT_PROFILES)) + raise ValueError(f"Unknown context profile: {self.profile}. Valid profiles: {valid}") + + +@dataclass(slots=True) +class SearchHit: + id: str + type: str + label: str + qualified_name: str = "" + path: str = "" + span: dict[str, int] = field(default_factory=dict) + score: float = 0.0 + summary: str = "" + context: list[ContextNode] = field(default_factory=list) + index_order: int = 0 + + def as_dict(self) -> dict[str, Any]: + return { + "id": self.id, + "type": self.type, + "label": self.label, + "qualified_name": self.qualified_name, + "path": self.path, + "span": dict(self.span), + "score": self.score, + "summary": self.summary, + "context": [node.as_dict() for node in self.context], + } + + +@dataclass(frozen=True, slots=True) +class CompactContextPayload: + query: str + profile: str + limit: int + budget: int + results: tuple[SearchHit, ...] + + def as_dict(self) -> dict[str, Any]: + return { + "query": self.query, + "profile": self.profile, + "limit": self.limit, + "budget": self.budget, + "results": [hit.as_dict() for hit in self.results], + } + + +@dataclass(frozen=True, slots=True) +class FTSIndexSpec: + node_type: str + index_name: str + order: int + + +class SearchService: + def __init__(self, store: Any) -> None: + self.store = store + self.indexes = tuple(_fts_index_specs()) + + def search(self, request: SearchRequest) -> CompactContextPayload: + request.validate() + hits = self._rank_hits(self._query_fts(request.query, request.limit)) + context_builder = CompactContextBuilder(self.store) + compact_hits: list[SearchHit] = [] + for hit in hits[: request.limit]: + hit.context = context_builder.build( + hit.id, + hit.type, + profile=request.profile, + limit=DEFAULT_CONTEXT_LIMIT, + budget=request.budget, + max_depth=request.max_depth, + ) + compact_hits.append(hit) + return CompactContextPayload( + query=request.query, + profile=request.profile, + limit=request.limit, + budget=request.budget, + results=tuple(compact_hits), + ) + + def _query_fts(self, query: str, limit: int) -> list[SearchHit]: + hits: list[SearchHit] = [] + for spec in self.indexes: + result = self.store.execute( + _fts_query_statement(spec), + {"query": query, "top": limit}, + ) + rows = result.get_all() + hits.extend(_hit_from_row(row, spec) for row in rows) + return hits + + def _rank_hits(self, hits: list[SearchHit]) -> list[SearchHit]: + best_by_id: dict[str, SearchHit] = {} + for hit in hits: + previous = best_by_id.get(hit.id) + if previous is None or _hit_sort_key(hit) < _hit_sort_key(previous): + best_by_id[hit.id] = hit + return sorted(best_by_id.values(), key=_hit_sort_key) + + +def _fts_query_statement(spec: FTSIndexSpec) -> str: + return ( + f"CALL QUERY_FTS_INDEX('{spec.node_type}', '{spec.index_name}', $query, TOP := $top) " + "RETURN node.id, node.label, node.qualified_name, node.path, " + "node.line_start, node.line_end, node.summary, score" + ) + + +def _fts_index_specs() -> list[FTSIndexSpec]: + specs: list[FTSIndexSpec] = [] + order = 0 + for index in SEARCH_INDEXES: + index_name = str(index["name"]) + for node_type in index["node_types"]: + specs.append(FTSIndexSpec(node_type=str(node_type), index_name=f"{index_name}_{node_type}", order=order)) + order += 1 + return specs + + +def _hit_from_row(row: Any, spec: FTSIndexSpec) -> SearchHit: + return SearchHit( + id=_text(_value(row, 0)), + type=spec.node_type, + label=_text(_value(row, 1)), + qualified_name=_text(_value(row, 2)), + path=_text(_value(row, 3)), + span=_span(_value(row, 4), _value(row, 5)), + summary=_text(_value(row, 6)), + score=float(_value(row, 7) or 0.0), + index_order=spec.order, + ) + + +def _hit_sort_key(hit: SearchHit) -> tuple[float, int, str, str, str]: + return (-hit.score, hit.index_order, hit.type, hit.path, hit.label) + + +def _span(line_start: Any, line_end: Any) -> dict[str, int]: + span: dict[str, int] = {} + if line_start is not None: + span["line_start"] = int(line_start) + if line_end is not None: + span["line_end"] = int(line_end) + return span + + +def _text(value: Any) -> str: + return "" if value is None else str(value) + + +def _value(row: Any, index: int) -> Any: + try: + return row[index] + except IndexError: + return None + + +__all__ = [ + "CompactContextPayload", + "DEFAULT_SEARCH_LIMIT", + "FTSIndexSpec", + "SearchHit", + "SearchRequest", + "SearchService", +] diff --git a/src/tests/test_search.py b/src/tests/test_search.py new file mode 100644 index 0000000..0fe3990 --- /dev/null +++ b/src/tests/test_search.py @@ -0,0 +1,194 @@ +from __future__ import annotations + +import json +import shutil +from pathlib import Path +from typing import Any + +import pytest + +from cli import main as cli_main +from ingest import GraphMaterializer +from reasoning import CompactContextBuilder +from retrieval.search import SearchHit, SearchRequest, SearchService + + +class _Result: + def __init__(self, rows: list[list[Any]]) -> None: + self.rows = rows + + def get_all(self) -> list[list[Any]]: + return self.rows + + +class _RecordingStore: + def __init__(self, rows: list[list[Any]] | None = None) -> None: + self.rows = rows or [] + self.calls: list[tuple[str, dict[str, Any] | None]] = [] + + def execute(self, statement: str, parameters: dict[str, Any] | None = None) -> _Result: + self.calls.append((statement, parameters)) + return _Result(self.rows) + + +def test_search_query_uses_ontology_index_names_and_parameterized_user_text() -> None: + malicious_query = "SampleService'); MATCH (n) RETURN n" + store = _RecordingStore() + + SearchService(store).search(SearchRequest(malicious_query, limit=2, budget=0)) + + assert store.calls + for statement, parameters in store.calls: + assert statement.startswith("CALL QUERY_FTS_INDEX('") + assert malicious_query not in statement + assert parameters == {"query": malicious_query, "top": 2} + + +def test_search_result_ranking_dedupes_by_node_id_and_uses_stable_tiebreaks() -> None: + service = SearchService(_RecordingStore()) + hits = [ + SearchHit(id="Function:helper", type="Function", label="helper", score=0.2, index_order=4), + SearchHit(id="Function:helper", type="Function", label="helper", score=0.7, index_order=4), + SearchHit(id="Class:SampleService", type="Class", label="SampleService", score=0.7, index_order=2), + SearchHit(id="File:service", type="File", label="service.py", path="service.py", score=0.1, index_order=8), + ] + + ranked = service._rank_hits(hits) + + assert [hit.id for hit in ranked] == ["Class:SampleService", "Function:helper", "File:service"] + assert ranked[1].score == 0.7 + + +def test_compact_context_respects_max_depth_limit_and_budget() -> None: + long_summary = "x" * 200 + store = _RecordingStore( + [["Method:run", "run", "sample.SampleService.run", "sample_project/service.py", 4, 6, long_summary]] + ) + builder = CompactContextBuilder(store) + + assert builder.build("Class:SampleService", "Class", profile="definitions", max_depth=0) == [] + + context = builder.build( + "Class:SampleService", + "Class", + profile="definitions", + limit=1, + budget=80, + max_depth=1, + ) + + assert len(context) == 1 + assert context[0].relation == "Defines" + assert context[0].direction == "outgoing" + assert context[0].summary + assert len(context[0].summary) < len(long_summary) + + +def test_search_request_rejects_invalid_profile() -> None: + with pytest.raises(ValueError, match="Unknown context profile"): + SearchRequest("SampleService", profile="missing").validate() + + +def test_search_service_returns_sample_class_with_compact_context(tmp_path: Path) -> None: + _require_graph_runtime() + materializer = _materialize_fixture(tmp_path, include_fts=True) + + payload = SearchService(materializer.store).search(SearchRequest("SampleService", limit=3)) + data = payload.as_dict() + + assert data["query"] == "SampleService" + assert data["profile"] == "brief" + class_hit = next(hit for hit in data["results"] if hit["type"] == "Class" and hit["label"] == "SampleService") + assert class_hit["path"] == "sample_project/service.py" + assert class_hit["score"] > 0 + assert class_hit["context"] + assert any(item["type"] in {"Module", "Method"} for item in class_hit["context"]) + + +def test_search_service_returns_function_hit_with_score_and_context(tmp_path: Path) -> None: + _require_graph_runtime() + materializer = _materialize_fixture(tmp_path, include_fts=True) + + payload = SearchService(materializer.store).search(SearchRequest("helper", limit=3)) + helper_hit = next(hit for hit in payload.as_dict()["results"] if hit["type"] == "Function" and hit["label"] == "helper") + + assert helper_hit["path"] == "sample_project/service.py" + assert helper_hit["span"]["line_start"] > 0 + assert helper_hit["score"] > 0 + assert helper_hit["context"] + + +def test_cli_search_and_context_return_compact_json_without_refresh(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + _require_graph_runtime() + source_root = _copy_fixture(tmp_path) + db_path = tmp_path / "graph.lbug" + manifest_path = tmp_path / "manifest.json" + + assert cli_main([ + "materialize", + "--source-root", + source_root.as_posix(), + "--db", + db_path.as_posix(), + "--manifest", + manifest_path.as_posix(), + "--mode", + "full", + ]) == 0 + capsys.readouterr() + + assert cli_main([ + "search", + "SampleService", + "--source-root", + source_root.as_posix(), + "--db", + db_path.as_posix(), + "--manifest", + manifest_path.as_posix(), + "--no-refresh", + "--json", + ]) == 0 + search_payload = json.loads(capsys.readouterr().out) + assert search_payload["results"] + assert any(hit["label"] == "SampleService" for hit in search_payload["results"]) + + assert cli_main([ + "context", + "helper", + "--source-root", + source_root.as_posix(), + "--db", + db_path.as_posix(), + "--manifest", + manifest_path.as_posix(), + "--no-refresh", + ]) == 0 + context_payload = json.loads(capsys.readouterr().out) + assert context_payload["results"] + assert any(hit["label"] == "helper" and hit["context"] for hit in context_payload["results"]) + + +def _require_graph_runtime() -> None: + pytest.importorskip("tree_sitter") + pytest.importorskip("tree_sitter_python") + pytest.importorskip("real_ladybug") + + +def _materialize_fixture(tmp_path: Path, *, include_fts: bool) -> GraphMaterializer: + source_root = _copy_fixture(tmp_path) + materializer = GraphMaterializer( + source_root, + db_path=":memory:", + manifest_path=tmp_path / "manifest.json", + include_fts=include_fts, + ) + materializer.materialize(mode="full") + return materializer + + +def _copy_fixture(tmp_path: Path) -> Path: + source = Path("tests/fixtures/sample_project") + target = tmp_path / "sample_project" + shutil.copytree(source, target) + return target From f31ae05d195de52f83fca6e038faa1489cc6dcf4 Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Fri, 22 May 2026 12:18:33 +0930 Subject: [PATCH 08/53] feat: Enhance search ranking --- src/retrieval/search.py | 137 +++++++++++++++++++++++++++++++++++++-- src/tests/test_search.py | 114 +++++++++++++++++++++++++++++++- 2 files changed, 244 insertions(+), 7 deletions(-) diff --git a/src/retrieval/search.py b/src/retrieval/search.py index 32d5ef6..dd24fe2 100644 --- a/src/retrieval/search.py +++ b/src/retrieval/search.py @@ -8,6 +8,10 @@ DEFAULT_SEARCH_LIMIT = 3 +MAX_CANDIDATE_LIMIT = 50 +MIN_CANDIDATE_LIMIT = 10 +DEFINITION_TYPES = {"Class", "Function", "Method", "Variable", "Constant"} +GENERIC_TYPES = {"Symbol", "Dependency"} @dataclass(frozen=True, slots=True) @@ -41,6 +45,8 @@ class SearchHit: path: str = "" span: dict[str, int] = field(default_factory=dict) score: float = 0.0 + rank_score: float = 0.0 + score_components: dict[str, float] = field(default_factory=dict) summary: str = "" context: list[ContextNode] = field(default_factory=list) index_order: int = 0 @@ -54,6 +60,8 @@ def as_dict(self) -> dict[str, Any]: "path": self.path, "span": dict(self.span), "score": self.score, + "rank_score": self.rank_score, + "score_components": dict(self.score_components), "summary": self.summary, "context": [node.as_dict() for node in self.context], } @@ -91,7 +99,12 @@ def __init__(self, store: Any) -> None: def search(self, request: SearchRequest) -> CompactContextPayload: request.validate() - hits = self._rank_hits(self._query_fts(request.query, request.limit)) + candidate_limit = _candidate_limit(request.limit) + hits = self._rank_hits( + self._query_fts(request.query, candidate_limit), + query=request.query, + profile=request.profile, + ) context_builder = CompactContextBuilder(self.store) compact_hits: list[SearchHit] = [] for hit in hits[: request.limit]: @@ -123,13 +136,15 @@ def _query_fts(self, query: str, limit: int) -> list[SearchHit]: hits.extend(_hit_from_row(row, spec) for row in rows) return hits - def _rank_hits(self, hits: list[SearchHit]) -> list[SearchHit]: + def _rank_hits(self, hits: list[SearchHit], *, query: str = "", profile: str = "brief") -> list[SearchHit]: best_by_id: dict[str, SearchHit] = {} for hit in hits: previous = best_by_id.get(hit.id) - if previous is None or _hit_sort_key(hit) < _hit_sort_key(previous): + if previous is None or _raw_hit_sort_key(hit) < _raw_hit_sort_key(previous): best_by_id[hit.id] = hit - return sorted(best_by_id.values(), key=_hit_sort_key) + deduped = list(best_by_id.values()) + _assign_rank_scores(deduped, query=query, profile=profile) + return sorted(deduped, key=_ranked_hit_sort_key) def _fts_query_statement(spec: FTSIndexSpec) -> str: @@ -165,7 +180,118 @@ def _hit_from_row(row: Any, spec: FTSIndexSpec) -> SearchHit: ) -def _hit_sort_key(hit: SearchHit) -> tuple[float, int, str, str, str]: +def _assign_rank_scores(hits: list[SearchHit], *, query: str, profile: str) -> None: + if not hits: + return + max_score = max((hit.score for hit in hits), default=0.0) + concrete_labels = { + _normalize(hit.label) + for hit in hits + if hit.type in DEFINITION_TYPES and hit.label + } + intent = _query_intent(query, profile) + + for hit in hits: + fts_score = hit.score / max_score if max_score > 0 else 0.0 + lexical_score = _lexical_score(query, hit) + type_score = _type_score(hit.type, intent) + generic_penalty = _generic_penalty(hit, concrete_labels) + rank_score = (0.45 * fts_score) + (0.35 * lexical_score) + type_score - generic_penalty + hit.score_components = { + "fts": round(fts_score, 6), + "lexical": round(lexical_score, 6), + "type": round(type_score, 6), + "generic_penalty": round(generic_penalty, 6), + } + hit.rank_score = round(rank_score, 6) + + +def _candidate_limit(limit: int) -> int: + return min(max(limit * 4, MIN_CANDIDATE_LIMIT), MAX_CANDIDATE_LIMIT) + + +def _query_intent(query: str, profile: str) -> str: + if profile in {"dependencies", "runtime", "docs"}: + return profile + if _looks_like_path(query): + return "path" + if _looks_like_identifier(query): + return "definition" + return "general" + + +def _lexical_score(query: str, hit: SearchHit) -> float: + normalized_query = _normalize(query) + if not normalized_query: + return 0.0 + label = _normalize(hit.label) + qualified_name = _normalize(hit.qualified_name) + path = _normalize(hit.path) + + if label == normalized_query: + return 1.0 + if qualified_name == normalized_query: + return 0.95 + if qualified_name.endswith(f".{normalized_query}") or qualified_name.endswith(f"/{normalized_query}"): + return 0.85 + if path == normalized_query or path.endswith(f"/{normalized_query}"): + return 0.8 + if normalized_query in label: + return 0.55 + if normalized_query in qualified_name: + return 0.45 + if normalized_query in path: + return 0.35 + return 0.0 + + +def _type_score(node_type: str, intent: str) -> float: + if intent == "definition": + if node_type in {"Class", "Function", "Method"}: + return 0.7 + if node_type in {"Variable", "Constant"}: + return 0.6 + if node_type == "Module": + return 0.2 + return 0.0 + if intent == "path": + return {"File": 0.7, "Module": 0.6, "SourceRoot": 0.25, "Repository": 0.2}.get(node_type, 0.0) + if intent == "dependencies": + return {"Dependency": 0.7, "ImportDeclaration": 0.65, "Module": 0.2}.get(node_type, 0.0) + if intent == "runtime": + return {"APIEndpoint": 0.7, "Route": 0.65, "Component": 0.55, "Query": 0.45, "SecretRef": 0.35}.get(node_type, 0.0) + if intent == "docs": + return {"DocumentationSource": 0.7, "DocumentationChunk": 0.65}.get(node_type, 0.0) + if node_type in DEFINITION_TYPES: + return 0.25 + return 0.0 + + +def _generic_penalty(hit: SearchHit, concrete_labels: set[str]) -> float: + if hit.type in GENERIC_TYPES and _normalize(hit.label) in concrete_labels: + return 0.45 + return 0.0 + + +def _looks_like_identifier(query: str) -> bool: + cleaned = query.strip() + return cleaned.replace("_", "").isalnum() and not cleaned[0:1].isdigit() + + +def _looks_like_path(query: str) -> bool: + cleaned = query.strip() + return "/" in cleaned or "\\" in cleaned or cleaned.endswith((".py", ".toml", ".md", ".json", ".yaml", ".yml")) + + +def _normalize(value: str) -> str: + return value.strip().lower() + + +def _ranked_hit_sort_key(hit: SearchHit) -> tuple[float, int, str, str, str]: + return (-hit.rank_score, hit.index_order, hit.type, hit.path, hit.label) + + +def _raw_hit_sort_key(hit: SearchHit) -> tuple[float, int, str, str, str]: return (-hit.score, hit.index_order, hit.type, hit.path, hit.label) @@ -193,6 +319,7 @@ def _value(row: Any, index: int) -> Any: "CompactContextPayload", "DEFAULT_SEARCH_LIMIT", "FTSIndexSpec", + "MAX_CANDIDATE_LIMIT", "SearchHit", "SearchRequest", "SearchService", diff --git a/src/tests/test_search.py b/src/tests/test_search.py index 0fe3990..2db267b 100644 --- a/src/tests/test_search.py +++ b/src/tests/test_search.py @@ -41,10 +41,10 @@ def test_search_query_uses_ontology_index_names_and_parameterized_user_text() -> for statement, parameters in store.calls: assert statement.startswith("CALL QUERY_FTS_INDEX('") assert malicious_query not in statement - assert parameters == {"query": malicious_query, "top": 2} + assert parameters == {"query": malicious_query, "top": 10} -def test_search_result_ranking_dedupes_by_node_id_and_uses_stable_tiebreaks() -> None: +def test_search_result_ranking_dedupes_by_node_id_preserving_best_raw_score() -> None: service = SearchService(_RecordingStore()) hits = [ SearchHit(id="Function:helper", type="Function", label="helper", score=0.2, index_order=4), @@ -59,6 +59,81 @@ def test_search_result_ranking_dedupes_by_node_id_and_uses_stable_tiebreaks() -> assert ranked[1].score == 0.7 +def test_identifier_query_reranks_concrete_definition_above_generic_symbol() -> None: + service = SearchService(_RecordingStore()) + hits = [ + SearchHit( + id="Symbol:SampleService", + type="Symbol", + label="SampleService", + path="sample_project/cli.py", + score=1.4, + index_order=0, + ), + SearchHit( + id="Class:SampleService", + type="Class", + label="SampleService", + qualified_name="sample_project.service.SampleService", + path="sample_project/service.py", + score=0.2, + index_order=2, + ), + ] + + ranked = service._rank_hits(hits, query="SampleService", profile="brief") + + assert ranked[0].type == "Class" + assert ranked[0].score == 0.2 + assert ranked[0].rank_score > ranked[1].rank_score + assert ranked[1].score_components["generic_penalty"] > 0 + + +def test_generic_penalty_only_applies_when_matching_concrete_definition_exists() -> None: + service = SearchService(_RecordingStore()) + + without_definition = service._rank_hits( + [SearchHit(id="Symbol:SampleService", type="Symbol", label="SampleService", score=1.0)], + query="SampleService", + profile="brief", + ) + with_definition = service._rank_hits( + [ + SearchHit(id="Symbol:SampleService", type="Symbol", label="SampleService", score=1.0), + SearchHit(id="Class:SampleService", type="Class", label="SampleService", score=0.1), + ], + query="SampleService", + profile="brief", + ) + + assert without_definition[0].score_components["generic_penalty"] == 0 + symbol_hit = next(hit for hit in with_definition if hit.type == "Symbol") + assert symbol_hit.score_components["generic_penalty"] > 0 + + +def test_path_and_dependency_intents_boost_matching_ontology_families() -> None: + service = SearchService(_RecordingStore()) + path_ranked = service._rank_hits( + [ + SearchHit(id="Function:helper", type="Function", label="helper", path="sample_project/service.py", score=1.0), + SearchHit(id="File:service", type="File", label="service.py", path="sample_project/service.py", score=0.3), + ], + query="service.py", + profile="brief", + ) + dependency_ranked = service._rank_hits( + [ + SearchHit(id="Class:SampleService", type="Class", label="SampleService", score=1.0), + SearchHit(id="Dependency:service", type="Dependency", label=".service.SampleService", score=0.3), + ], + query=".service.SampleService", + profile="dependencies", + ) + + assert path_ranked[0].type == "File" + assert dependency_ranked[0].type == "Dependency" + + def test_compact_context_respects_max_depth_limit_and_budget() -> None: long_summary = "x" * 200 store = _RecordingStore( @@ -101,10 +176,26 @@ def test_search_service_returns_sample_class_with_compact_context(tmp_path: Path class_hit = next(hit for hit in data["results"] if hit["type"] == "Class" and hit["label"] == "SampleService") assert class_hit["path"] == "sample_project/service.py" assert class_hit["score"] > 0 + assert class_hit["rank_score"] > 0 + assert class_hit["score_components"]["type"] > 0 assert class_hit["context"] assert any(item["type"] in {"Module", "Method"} for item in class_hit["context"]) +def test_search_service_returns_sample_class_first_for_exact_identifier(tmp_path: Path) -> None: + _require_graph_runtime() + materializer = _materialize_fixture(tmp_path, include_fts=True) + + payload = SearchService(materializer.store).search(SearchRequest("SampleService", limit=1)) + hit = payload.as_dict()["results"][0] + + assert hit["type"] == "Class" + assert hit["label"] == "SampleService" + assert hit["path"] == "sample_project/service.py" + assert hit["score"] < 1.0 + assert hit["rank_score"] > hit["score"] + + def test_search_service_returns_function_hit_with_score_and_context(tmp_path: Path) -> None: _require_graph_runtime() materializer = _materialize_fixture(tmp_path, include_fts=True) @@ -115,6 +206,7 @@ def test_search_service_returns_function_hit_with_score_and_context(tmp_path: Pa assert helper_hit["path"] == "sample_project/service.py" assert helper_hit["span"]["line_start"] > 0 assert helper_hit["score"] > 0 + assert helper_hit["rank_score"] > 0 assert helper_hit["context"] @@ -153,6 +245,24 @@ def test_cli_search_and_context_return_compact_json_without_refresh(tmp_path: Pa assert search_payload["results"] assert any(hit["label"] == "SampleService" for hit in search_payload["results"]) + assert cli_main([ + "search", + "SampleService", + "--source-root", + source_root.as_posix(), + "--db", + db_path.as_posix(), + "--manifest", + manifest_path.as_posix(), + "--limit", + "1", + "--no-refresh", + "--json", + ]) == 0 + top_payload = json.loads(capsys.readouterr().out) + assert top_payload["results"][0]["type"] == "Class" + assert top_payload["results"][0]["rank_score"] > top_payload["results"][0]["score"] + assert cli_main([ "context", "helper", From b0f4a6e98808b5a9928f31736190ab944be31843 Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Fri, 22 May 2026 15:42:28 +0930 Subject: [PATCH 09/53] feat: speed up graph materialization --- README.md | 2 +- src/db/store.py | 308 ++++++++++++++++++++++++++------ src/extract/graph_builder.py | 9 +- src/ingest/materializer.py | 297 +++++++++++++++++++++++++----- src/ontology/ontology.py | 8 +- src/tests/test_graph_builder.py | 36 ++++ src/tests/test_materializer.py | 162 +++++++++++++++++ src/tests/test_ontology.py | 17 ++ src/tests/test_schema.py | 26 +++ 9 files changed, 761 insertions(+), 104 deletions(-) diff --git a/README.md b/README.md index 0157ba0..6be8c38 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # codebaseGraph -`codebaseGraph` is a generic project/code knowledge graph engine for coding repositories. It scans a source root, builds a typed graph of files, modules, symbols, imports, calls, dependencies, entry points, and documentation sources, and exposes search, compact context, schema, and read-only query helpers. +`codebaseGraph` is a generic project/code knowledge graph engine for coding repositories. The current filesystem materializer scans Python `.py` files, builds a typed graph of files, modules, symbols, imports, calls, dependencies, and entry points, and exposes search, compact context, schema, and read-only query helpers. The ontology and graph builder can represent documentation chunks from normalized parser or capture input, but Markdown and other documentation files are not scanned by the materializer yet. ## Install for local development diff --git a/src/db/store.py b/src/db/store.py index 3cdd577..651af13 100644 --- a/src/db/store.py +++ b/src/db/store.py @@ -1,11 +1,15 @@ from __future__ import annotations +import csv import json +import tempfile +from collections import defaultdict from collections.abc import Mapping +from dataclasses import dataclass from pathlib import Path from typing import Any -from core import CodeGraph, GraphEdge, GraphNode +from core import CodeGraph from ontology import NODE_TYPES, RELATION_TYPES from .schema import build_ladybug_schema, build_ladybug_schema_statements, quote_identifier @@ -15,6 +19,14 @@ class LadybugUnavailableError(RuntimeError): pass +@dataclass(frozen=True, slots=True) +class BulkLoadStats: + node_rows: int = 0 + edge_rows: int = 0 + connector_rows: int = 0 + copy_calls: int = 0 + + class LadybugCodeGraphStore: def __init__(self, db_path: str | Path = ":memory:", *, include_fts: bool = True) -> None: self.db_path = db_path @@ -45,6 +57,10 @@ def execute(self, statement: str, parameters: dict[str, Any] | None = None) -> A return self.conn.execute(statement) return self.conn.execute(statement, parameters) + def close(self) -> None: + self.conn.close() + self.db.close() + def clear_graph(self) -> None: for relation_type in RELATION_TYPES: self._execute_ignoring_missing(f"MATCH ()-[r:{quote_identifier(f'FROM_{relation_type.name}')}]->() DELETE r") @@ -61,18 +77,47 @@ def replace_partition( *, previous_entry: Mapping[str, Any] | Any | None = None, retained_node_ids: set[str] | None = None, + retained_edge_ids: set[str] | None = None, ) -> None: if previous_entry is not None: - self.delete_partition(path, manifest_entry=previous_entry, retained_node_ids=retained_node_ids) + self.delete_partition( + path, + manifest_entry=previous_entry, + retained_node_ids=retained_node_ids, + retained_edge_ids=retained_edge_ids, + ) + + self.insert_graphs_bulk( + [graph], + skip_node_ids=retained_node_ids, + skip_edge_ids=retained_edge_ids, + ) - for node in graph.nodes.values(): - self._upsert_node(node) - for edge in graph.edges.values(): - self._upsert_edge_node(edge) - for edge in graph.edges.values(): - source = graph.nodes[edge.source_id] - target = graph.nodes[edge.target_id] - self._upsert_connector(edge, source, target) + def insert_graphs_bulk( + self, + graphs: list[CodeGraph] | tuple[CodeGraph, ...], + *, + skip_node_ids: set[str] | None = None, + skip_edge_ids: set[str] | None = None, + ) -> BulkLoadStats: + staging_tables = _build_bulk_staging_tables( + graphs, + skip_node_ids=skip_node_ids, + skip_edge_ids=skip_edge_ids, + ) + if staging_tables.is_empty: + return BulkLoadStats() + + with tempfile.TemporaryDirectory(prefix="codebase-graph-ladybug-") as staging_dir: + staging = staging_tables.write(Path(staging_dir)) + for statement in staging.copy_statements: + self.execute(statement) + return BulkLoadStats( + node_rows=staging.node_rows, + edge_rows=staging.edge_rows, + connector_rows=staging.connector_rows, + copy_calls=len(staging.copy_statements), + ) def delete_partition( self, @@ -80,14 +125,18 @@ def delete_partition( *, manifest_entry: Mapping[str, Any] | Any | None = None, retained_node_ids: set[str] | None = None, + retained_edge_ids: set[str] | None = None, ) -> None: if manifest_entry is None: return retained = retained_node_ids or set() + retained_edges = retained_edge_ids or set() edge_types = _entry_mapping(manifest_entry, "edge_types") node_types = _entry_mapping(manifest_entry, "node_types") for edge_id in _entry_values(manifest_entry, "edge_ids"): + if edge_id in retained_edges: + continue edge_type = edge_types.get(edge_id) if edge_type: self._delete_edge(edge_id, edge_type) @@ -123,38 +172,6 @@ def _execute_ignoring_missing(self, statement: str, parameters: dict[str, Any] | if "does not exist" not in message and "not found" not in message: raise - def _upsert_node(self, node: GraphNode) -> None: - table_fields = NODE_FIELDS[node.table] - row = node.as_dict() - statement, parameters = _merge_statement(node.table, table_fields, row) - self.execute(statement, parameters) - - def _upsert_edge_node(self, edge: GraphEdge) -> None: - table_fields = EDGE_FIELDS_BY_TYPE[edge.type] - row = edge.as_dict() - statement, parameters = _merge_statement(edge.type, table_fields, row) - self.execute(statement, parameters) - - def _upsert_connector(self, edge: GraphEdge, source: GraphNode, target: GraphNode) -> None: - from_relation = quote_identifier(f"FROM_{edge.type}") - to_relation = quote_identifier(f"TO_{edge.type}") - self.execute( - ( - f"MATCH (source:{quote_identifier(source.table)} {{id: $source_id}}), " - f"(edge:{quote_identifier(edge.type)} {{id: $edge_id}}) " - f"MERGE (source)-[:{from_relation}]->(edge)" - ), - {"source_id": source.id, "edge_id": edge.id}, - ) - self.execute( - ( - f"MATCH (edge:{quote_identifier(edge.type)} {{id: $edge_id}}), " - f"(target:{quote_identifier(target.table)} {{id: $target_id}}) " - f"MERGE (edge)-[:{to_relation}]->(target)" - ), - {"edge_id": edge.id, "target_id": target.id}, - ) - def _delete_edge(self, edge_id: str, edge_type: str) -> None: self._execute_ignoring_missing( f"MATCH ()-[r:{quote_identifier(f'FROM_{edge_type}')}]->(edge:{quote_identifier(edge_type)} {{id: $id}}) DELETE r", @@ -193,22 +210,197 @@ def create_ladybug_database(db_path: str | Path = ":memory:", *, include_fts: bo } -def _merge_statement(table: str, fields: tuple[Any, ...], row: Mapping[str, Any]) -> tuple[str, dict[str, Any]]: - parameters: dict[str, Any] = {"id": _field_value("id", row, "string")} - assignments = [] - for field in fields: - if field.name == "id": - continue - field_value = _field_value(field.name, row, field.value_type) - if field_value is None: - continue - parameters[field.name] = field_value - value = f"CAST(${field.name} AS JSON)" if field.value_type == "json" else f"${field.name}" - assignments.append(f"n.{quote_identifier(field.name)} = {value}") - statement = f"MERGE (n:{quote_identifier(table)} {{id: $id}})" - if assignments: - statement += f" SET {', '.join(assignments)}" - return statement, parameters +@dataclass(slots=True) +class _BulkStagingTables: + nodes: dict[str, dict[str, dict[str, Any]]] + edges: dict[str, dict[str, dict[str, Any]]] + connectors: dict[tuple[str, str, str], dict[tuple[str, str, str], dict[str, str]]] + + @property + def is_empty(self) -> bool: + return not any(self.nodes.values()) and not any(self.edges.values()) and not any(self.connectors.values()) + + def write(self, staging_dir: Path) -> _BulkStagingResult: + staging_dir.mkdir(parents=True, exist_ok=True) + copy_statements: list[str] = [] + node_rows = 0 + edge_rows = 0 + connector_rows = 0 + + for node_type in NODE_TYPES: + rows = self.nodes.get(node_type.name, {}) + if not rows: + continue + path = staging_dir / f"{_stage_file_stem(node_type.name)}.json" + _write_json_rows(path, rows.values()) + node_rows += len(rows) + copy_statements.append(f'COPY {quote_identifier(node_type.name)} FROM "{_copy_path(path)}";') + + for relation_type in RELATION_TYPES: + rows = self.edges.get(relation_type.name, {}) + if not rows: + continue + path = staging_dir / f"{_stage_file_stem(relation_type.name)}.json" + _write_json_rows(path, rows.values()) + edge_rows += len(rows) + copy_statements.append(f'COPY {quote_identifier(relation_type.name)} FROM "{_copy_path(path)}";') + + for relation_type in RELATION_TYPES: + for connector_table in (f"FROM_{relation_type.name}", f"TO_{relation_type.name}"): + connector_groups = [ + (endpoint_pair, rows) + for endpoint_pair, rows in self.connectors.items() + if endpoint_pair[0] == connector_table and rows + ] + for (table, source_type, target_type), rows in sorted(connector_groups): + path = staging_dir / ( + f"{_stage_file_stem(table)}__" + f"{_stage_file_stem(source_type)}__{_stage_file_stem(target_type)}.csv" + ) + _write_csv_rows(path, ("from_id", "to_id", "role"), rows.values()) + connector_rows += len(rows) + copy_statements.append( + f'COPY {quote_identifier(table)} FROM "{_copy_path(path)}" ' + f'(header=true, from="{source_type}", to="{target_type}");' + ) + + return _BulkStagingResult( + copy_statements=tuple(copy_statements), + node_rows=node_rows, + edge_rows=edge_rows, + connector_rows=connector_rows, + ) + + +@dataclass(frozen=True, slots=True) +class _BulkStagingResult: + copy_statements: tuple[str, ...] + node_rows: int + edge_rows: int + connector_rows: int + + +def _build_bulk_staging_tables( + graphs: list[CodeGraph] | tuple[CodeGraph, ...], + *, + skip_node_ids: set[str] | None = None, + skip_edge_ids: set[str] | None = None, +) -> _BulkStagingTables: + skipped_nodes = skip_node_ids or set() + skipped_edges = skip_edge_ids or set() + node_rows: dict[str, dict[str, dict[str, Any]]] = defaultdict(dict) + edge_rows: dict[str, dict[str, dict[str, Any]]] = defaultdict(dict) + connector_rows: dict[tuple[str, str, str], dict[tuple[str, str, str], dict[str, str]]] = defaultdict(dict) + + for graph in graphs: + for node in graph.nodes.values(): + if node.id in skipped_nodes: + continue + row = _row_for_fields(node.as_dict(), NODE_FIELDS[node.table], for_json_copy=True) + _merge_staged_row(node_rows[node.table], node.id, row) + + for edge in graph.edges.values(): + if edge.id in skipped_edges: + continue + row = _row_for_fields(edge.as_dict(), EDGE_FIELDS_BY_TYPE[edge.type], for_json_copy=True) + _merge_staged_row(edge_rows[edge.type], edge.id, row) + + for edge in graph.edges.values(): + if edge.id in skipped_edges: + continue + source = graph.nodes[edge.source_id] + target = graph.nodes[edge.target_id] + _add_connector_row( + connector_rows, + table=f"FROM_{edge.type}", + source_type=source.table, + target_type=edge.type, + from_id=source.id, + to_id=edge.id, + role="source", + ) + _add_connector_row( + connector_rows, + table=f"TO_{edge.type}", + source_type=edge.type, + target_type=target.table, + from_id=edge.id, + to_id=target.id, + role="target", + ) + + return _BulkStagingTables(nodes=dict(node_rows), edges=dict(edge_rows), connectors=dict(connector_rows)) + + +def _row_for_fields(row: Mapping[str, Any], fields: tuple[Any, ...], *, for_json_copy: bool = False) -> dict[str, Any]: + return { + field.name: _copy_field_value(field.name, row, field.value_type, for_json_copy=for_json_copy) + for field in fields + } + + +def _copy_field_value(name: str, row: Mapping[str, Any], value_type: str, *, for_json_copy: bool = False) -> Any: + if not for_json_copy or value_type != "json": + return _field_value(name, row, value_type) + if name in row: + value = row[name] + else: + metadata = row.get("metadata") if isinstance(row.get("metadata"), Mapping) else {} + value = metadata.get(name) + safe = _json_safe(value if value is not None else {}) + if safe is _OMIT_JSON_VALUE: + return {} + return safe + + +def _merge_staged_row(rows: dict[str, dict[str, Any]], row_id: str, row: dict[str, Any]) -> None: + existing = rows.get(row_id) + if existing is None: + rows[row_id] = row + return + for key, value in row.items(): + if value not in (None, "", {}, []) and existing.get(key) in (None, "", {}, []): + existing[key] = value + existing_metadata = existing.get("metadata") + incoming_metadata = row.get("metadata") + if isinstance(existing_metadata, dict) and isinstance(incoming_metadata, dict): + existing_metadata.update(incoming_metadata) + + +def _add_connector_row( + rows: dict[tuple[str, str, str], dict[tuple[str, str, str], dict[str, str]]], + *, + table: str, + source_type: str, + target_type: str, + from_id: str, + to_id: str, + role: str, +) -> None: + key = (table, source_type, target_type) + rows[key][(from_id, to_id, role)] = {"from_id": from_id, "to_id": to_id, "role": role} + + +def _write_json_rows(path: Path, rows: Any) -> None: + with path.open("w", encoding="utf-8") as handle: + json.dump(list(rows), handle, separators=(",", ":"), sort_keys=True) + handle.write("\n") + + +def _write_csv_rows(path: Path, columns: tuple[str, ...], rows: Any) -> None: + with path.open("w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter(handle, fieldnames=columns, extrasaction="ignore") + writer.writeheader() + for row in rows: + writer.writerow({column: row.get(column, "") for column in columns}) + + +def _stage_file_stem(name: str) -> str: + return "".join(character.lower() if character.isalnum() else "_" for character in name).strip("_") or "table" + + +def _copy_path(path: Path) -> str: + return path.as_posix().replace('"', '\\"') def _field_value(name: str, row: Mapping[str, Any], value_type: str) -> Any: diff --git a/src/extract/graph_builder.py b/src/extract/graph_builder.py index ea3718f..749cad7 100644 --- a/src/extract/graph_builder.py +++ b/src/extract/graph_builder.py @@ -307,7 +307,7 @@ def _emit_import(self, node: ParserNode, owner: ScopeFrame, syntax_id: str) -> G metadata={"imported_name": imported}, ) self._connect_owner(owner, semantic) - self._edge("Imports", owner.node_id, semantic.id, "declares_import") + self._edge_if_allowed("Imports", _import_source_id(owner), semantic.id, "declares_import") self._derived_from(semantic.id, syntax_id) if imported: dependency = self._support_node("Dependency", imported, imported, path=self._context.path) @@ -890,6 +890,7 @@ def _semantic_children(self, node: ParserNode) -> tuple[Any, ...]: "Component", } RUNTIME_TARGET_TYPES = {"Function", "Method", "Component", "APIEndpoint"} +IMPORT_SOURCE_TYPES = {"File", "Module", "Scope"} DEFINED_CAPTURE_TABLES = { "APIEndpoint", "Component", @@ -997,6 +998,12 @@ def _table_from_capture(capture_name: str, owner: ScopeFrame) -> str | None: return None +def _import_source_id(owner: ScopeFrame) -> str: + if owner.table in IMPORT_SOURCE_TYPES: + return owner.node_id + return owner.scope_id or owner.node_id + + def _id(prefix: str, value: str) -> str: return f"{prefix}:{hashlib.sha1(value.encode('utf-8')).hexdigest()[:20]}" diff --git a/src/ingest/materializer.py b/src/ingest/materializer.py index 2cc3dfd..95f4dc2 100644 --- a/src/ingest/materializer.py +++ b/src/ingest/materializer.py @@ -3,6 +3,7 @@ import hashlib import json import os +import tempfile from collections.abc import Mapping from dataclasses import dataclass, field from datetime import UTC, datetime @@ -207,22 +208,35 @@ def __init__( ) -> None: self.source_root = Path(source_root).resolve() self.state_dir = self.source_root / DEFAULT_STATE_DIR - self.db_path = db_path if db_path is not None else self.state_dir / DEFAULT_DB_NAME + self.db_path = _normalize_db_path(db_path if db_path is not None else self.state_dir / DEFAULT_DB_NAME) self.manifest_path = Path(manifest_path) if manifest_path is not None else self.state_dir / DEFAULT_MANIFEST_NAME self.include_fts = include_fts self.repository_label = repository_label or self.source_root.name or "repository" - self.store = store or create_ladybug_database(self.db_path, include_fts=include_fts) + self._store = store + self._store_injected = store is not None self.builder = GraphBuilder(repository_label=self.repository_label, source_root=self.source_root) + @property + def store(self) -> LadybugCodeGraphStore: + if self._store is None: + self._store = create_ladybug_database(self.db_path, include_fts=self.include_fts) + return self._store + + @store.setter + def store(self, value: LadybugCodeGraphStore | None) -> None: + self._store = value + self._store_injected = value is not None + def materialize(self, mode: MaterializeMode = "changed") -> MaterializationResult: if mode not in {"full", "changed"}: raise ValueError(f"Unsupported materialization mode: {mode}") - previous_manifest = self.store.read_manifest(self.manifest_path) + previous_manifest = self._read_manifest() snapshots, diagnostics = self._scan_source_files() supported = {path: snapshot for path, snapshot in snapshots.items() if snapshot.language is not None} + force_atomic_recovery = self._should_force_atomic_recovery() - if mode == "full": + if mode == "full" or force_atomic_recovery: diff = ManifestDiff( added=tuple(sorted(supported)), modified=(), @@ -230,35 +244,67 @@ def materialize(self, mode: MaterializeMode = "changed") -> MaterializationResul deleted=tuple(sorted(previous_manifest.files)), force_rebuild=True, ) + if self._can_atomic_rebuild(): + return self._materialize_full_atomic( + mode=mode, + snapshots=snapshots, + diagnostics=diagnostics, + supported=supported, + diff=diff, + ) self.store.clear_graph() retained_node_ids: set[str] = set() + retained_edge_ids: set[str] = set() else: diff = previous_manifest.diff(supported) if diff.force_rebuild: + if self._can_atomic_rebuild(): + return self._materialize_full_atomic( + mode=mode, + snapshots=snapshots, + diagnostics=diagnostics, + supported=supported, + diff=diff, + ) self.store.clear_graph() retained_node_ids = set() + retained_edge_ids = set() else: - retained_node_ids = _retained_node_ids(previous_manifest, set(diff.rebuild_paths) | set(diff.deleted)) + touched_paths = set(diff.rebuild_paths) | set(diff.deleted) + retained_node_ids = _retained_node_ids(previous_manifest, touched_paths) + retained_edge_ids = _retained_edge_ids(previous_manifest, touched_paths) for path in diff.deleted: self.store.delete_partition( path, manifest_entry=previous_manifest.files.get(path), retained_node_ids=retained_node_ids, + retained_edge_ids=retained_edge_ids, ) rebuilt_entries: dict[str, ManifestEntry] = {} + rebuilt_graphs: dict[str, CodeGraph] = {} for path in diff.rebuild_paths: snapshot = supported[path] - previous_entry = None if diff.force_rebuild else previous_manifest.files.get(path) graph = self._build_graph(snapshot) - self.store.replace_partition( - path, - graph, - previous_entry=previous_entry, - retained_node_ids=retained_node_ids, - ) + rebuilt_graphs[path] = graph rebuilt_entries[path] = _manifest_entry(snapshot, graph) + if not diff.force_rebuild: + for path in diff.rebuild_paths: + self.store.delete_partition( + path, + manifest_entry=previous_manifest.files.get(path), + retained_node_ids=retained_node_ids, + retained_edge_ids=retained_edge_ids, + ) + + if rebuilt_graphs: + self.store.insert_graphs_bulk( + [rebuilt_graphs[path] for path in sorted(rebuilt_graphs)], + skip_node_ids=retained_node_ids, + skip_edge_ids=retained_edge_ids, + ) + next_files = { path: entry for path, entry in previous_manifest.files.items() @@ -266,40 +312,122 @@ def materialize(self, mode: MaterializeMode = "changed") -> MaterializationResul } next_files.update(rebuilt_entries) next_manifest = MaterializationManifest(files=next_files) - self.store.write_manifest(next_manifest, self.manifest_path) + self._write_manifest(next_manifest) + + return _materialization_result( + mode=mode, + snapshots=snapshots, + diagnostics=diagnostics, + diff=diff, + manifest_path=self.manifest_path, + rebuilt_entries=rebuilt_entries, + next_manifest=next_manifest, + ) + + def _materialize_full_atomic( + self, + *, + mode: MaterializeMode, + snapshots: Mapping[str, SourceSnapshot], + diagnostics: list[str], + supported: Mapping[str, SourceSnapshot], + diff: ManifestDiff, + ) -> MaterializationResult: + rebuilt_entries: dict[str, ManifestEntry] = {} + rebuilt_graphs: dict[str, CodeGraph] = {} + for path in diff.rebuild_paths: + snapshot = supported[path] + graph = self._build_graph(snapshot) + rebuilt_graphs[path] = graph + rebuilt_entries[path] = _manifest_entry(snapshot, graph) + + next_manifest = MaterializationManifest(files=rebuilt_entries) + target_db_path = _filesystem_db_path(self.db_path) + temp_db_path = _temporary_sibling(target_db_path, suffix=".lbug.tmp") + temp_manifest_path = _temporary_sibling(self.manifest_path, suffix=".manifest.tmp") + marker_path = self._rebuild_marker_path + temp_store: LadybugCodeGraphStore | None = None + try: + temp_store = create_ladybug_database(temp_db_path, include_fts=self.include_fts) + if rebuilt_graphs: + temp_store.insert_graphs_bulk([rebuilt_graphs[path] for path in sorted(rebuilt_graphs)]) + temp_store.close() + temp_store = None + + next_manifest.write(temp_manifest_path) + _write_rebuild_marker(marker_path, target_db_path, self.manifest_path) + self._close_store() + os.replace(temp_db_path, target_db_path) + os.replace(temp_manifest_path, self.manifest_path) + _unlink_if_exists(marker_path) + self._store = None + except Exception: + if temp_store is not None: + temp_store.close() + _unlink_if_exists(temp_db_path) + _unlink_if_exists(temp_manifest_path) + _unlink_if_exists(temp_manifest_path.with_suffix(temp_manifest_path.suffix + ".tmp")) + raise - unsupported_paths = tuple(path for path, snapshot in snapshots.items() if snapshot.language is None) - skipped_paths = tuple(sorted((*diff.unchanged, *unsupported_paths))) - return MaterializationResult( + return _materialization_result( mode=mode, - scanned=len(snapshots), - rebuilt=len(rebuilt_entries), - skipped=len(skipped_paths), - deleted=len(diff.deleted), - diagnostics=tuple(diagnostics), - manifest_path=self.manifest_path.as_posix(), - rebuilt_paths=tuple(sorted(rebuilt_entries)), - skipped_paths=skipped_paths, - deleted_paths=diff.deleted, - graph_summary=_manifest_summary(next_manifest), + snapshots=snapshots, + diagnostics=diagnostics, + diff=diff, + manifest_path=self.manifest_path, + rebuilt_entries=rebuilt_entries, + next_manifest=next_manifest, ) + def _read_manifest(self) -> MaterializationManifest: + if self._store_injected and self._store is not None and hasattr(self._store, "read_manifest"): + return self._store.read_manifest(self.manifest_path) + return MaterializationManifest.load(self.manifest_path) + + def _write_manifest(self, manifest: MaterializationManifest) -> None: + if self._store_injected and self._store is not None and hasattr(self._store, "write_manifest"): + self._store.write_manifest(manifest, self.manifest_path) + return + manifest.write(self.manifest_path) + + def _can_atomic_rebuild(self) -> bool: + return not self._store_injected and not _is_memory_db_path(self.db_path) + + def _should_force_atomic_recovery(self) -> bool: + return self._can_atomic_rebuild() and self._rebuild_marker_path.exists() + + @property + def _rebuild_marker_path(self) -> Path: + return self.manifest_path.with_suffix(self.manifest_path.suffix + ".rebuild-pending") + + def _close_store(self) -> None: + if self._store is None: + return + close = getattr(self._store, "close", None) + if callable(close): + close() + self._store = None + def _scan_source_files(self) -> tuple[dict[str, SourceSnapshot], list[str]]: snapshots: dict[str, SourceSnapshot] = {} diagnostics: list[str] = [] - for path in sorted(self.source_root.rglob("*")): - if not path.is_file() or _is_excluded(path, self.source_root): - continue - relative_path = path.relative_to(self.source_root).as_posix() - language = SUPPORTED_SUFFIXES.get(path.suffix) - snapshots[relative_path] = SourceSnapshot( - path=relative_path, - absolute_path=path, - content_hash=_file_hash(path), - language=language, - ) - if language is None: - diagnostics.append(f"Skipped unsupported file: {relative_path}") + for current_root, dirnames, filenames in os.walk(self.source_root): + dirnames[:] = [name for name in sorted(dirnames) if not _is_excluded_part(name)] + current_path = Path(current_root) + for filename in sorted(filenames): + path = current_path / filename + if _is_excluded(path, self.source_root): + continue + relative_path = path.relative_to(self.source_root).as_posix() + language = SUPPORTED_SUFFIXES.get(path.suffix) + snapshots[relative_path] = SourceSnapshot( + path=relative_path, + absolute_path=path, + content_hash=_file_hash(path), + language=language, + ) + if language is None: + diagnostics.append(f"Skipped unsupported file: {relative_path}") return snapshots, diagnostics def _build_graph(self, snapshot: SourceSnapshot) -> CodeGraph: @@ -320,9 +448,62 @@ def _build_graph(self, snapshot: SourceSnapshot) -> CodeGraph: return result.graph +def _is_excluded_part(part: str) -> bool: + return part in EXCLUDED_PARTS or part.endswith(".egg-info") + + +def _normalize_db_path(db_path: str | Path) -> str | Path: + if _is_memory_db_path(db_path): + return ":memory:" + return Path(db_path) + + +def _is_memory_db_path(db_path: str | Path) -> bool: + return str(db_path) == ":memory:" + + +def _filesystem_db_path(db_path: str | Path) -> Path: + if _is_memory_db_path(db_path): + raise ValueError("In-memory databases do not have a filesystem path") + return Path(db_path) + + +def _temporary_sibling(path: Path, *, suffix: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temp_path = tempfile.mkstemp(prefix=f".{path.name}.", suffix=suffix, dir=path.parent) + os.close(descriptor) + os.unlink(temp_path) + return Path(temp_path) + + +def _write_rebuild_marker(marker_path: Path, db_path: Path, manifest_path: Path) -> None: + marker_path.parent.mkdir(parents=True, exist_ok=True) + tmp_path = marker_path.with_suffix(marker_path.suffix + ".tmp") + with tmp_path.open("w", encoding="utf-8") as handle: + json.dump( + { + "created_at": datetime.now(UTC).isoformat(), + "db_path": db_path.as_posix(), + "manifest_path": manifest_path.as_posix(), + }, + handle, + indent=2, + sort_keys=True, + ) + handle.write("\n") + os.replace(tmp_path, marker_path) + + +def _unlink_if_exists(path: Path) -> None: + try: + path.unlink() + except FileNotFoundError: + return + + def _is_excluded(path: Path, source_root: Path) -> bool: parts = path.relative_to(source_root).parts - return any(part in EXCLUDED_PARTS or part.endswith(".egg-info") for part in parts) + return any(_is_excluded_part(part) for part in parts) def _file_hash(path: Path) -> str: @@ -351,6 +532,33 @@ def _manifest_entry(snapshot: SourceSnapshot, graph: CodeGraph) -> ManifestEntry ) +def _materialization_result( + *, + mode: MaterializeMode, + snapshots: Mapping[str, SourceSnapshot], + diagnostics: list[str], + diff: ManifestDiff, + manifest_path: Path, + rebuilt_entries: Mapping[str, ManifestEntry], + next_manifest: MaterializationManifest, +) -> MaterializationResult: + unsupported_paths = tuple(path for path, snapshot in snapshots.items() if snapshot.language is None) + skipped_paths = tuple(sorted((*diff.unchanged, *unsupported_paths))) + return MaterializationResult( + mode=mode, + scanned=len(snapshots), + rebuilt=len(rebuilt_entries), + skipped=len(skipped_paths), + deleted=len(diff.deleted), + diagnostics=tuple(diagnostics), + manifest_path=manifest_path.as_posix(), + rebuilt_paths=tuple(sorted(rebuilt_entries)), + skipped_paths=skipped_paths, + deleted_paths=diff.deleted, + graph_summary=_manifest_summary(next_manifest), + ) + + def _retained_node_ids(manifest: MaterializationManifest, touched_paths: set[str]) -> set[str]: retained: set[str] = set() for path, entry in manifest.files.items(): @@ -360,6 +568,15 @@ def _retained_node_ids(manifest: MaterializationManifest, touched_paths: set[str return retained +def _retained_edge_ids(manifest: MaterializationManifest, touched_paths: set[str]) -> set[str]: + retained: set[str] = set() + for path, entry in manifest.files.items(): + if path in touched_paths: + continue + retained.update(entry.edge_ids) + return retained + + def _manifest_summary(manifest: MaterializationManifest) -> dict[str, int | str]: node_ids: set[str] = set() edge_ids: set[str] = set() diff --git a/src/ontology/ontology.py b/src/ontology/ontology.py index 7744ca3..6958f88 100644 --- a/src/ontology/ontology.py +++ b/src/ontology/ontology.py @@ -710,25 +710,25 @@ def _relation( QueryHelperSpec( "callgraph_neighborhood", "Find call expressions and resolved callable targets near a symbol.", - "MATCH (c:CallExpression)-[:ResolvesTo]->(target) RETURN c.id, c.path, target.id, target.label LIMIT 50", + "MATCH (c:CallExpression)-[:FROM_ResolvesTo]->(:ResolvesTo)-[:TO_ResolvesTo]->(target) RETURN c.id, c.path, target.id, target.label LIMIT 50", returns=("call_id", "path", "target_id", "target_label"), ), QueryHelperSpec( "dependency_map", "Inspect imports and dependencies.", - "MATCH (i:ImportDeclaration)-[:DependsOn]->(d:Dependency) RETURN i.id, i.label, d.id, d.label LIMIT 100", + "MATCH (i:ImportDeclaration)-[:FROM_DependsOn]->(:DependsOn)-[:TO_DependsOn]->(d:Dependency) RETURN i.id, i.label, d.id, d.label LIMIT 100", returns=("import_id", "import_label", "dependency_id", "dependency_label"), ), QueryHelperSpec( "runtime_surface", "Inspect routes, endpoints, executed queries, and secret use.", - "MATCH (r:Route)-[:RoutesTo]->(e:APIEndpoint) RETURN r.id, r.label, e.id, e.label LIMIT 100", + "MATCH (r:Route)-[:FROM_RoutesTo]->(:RoutesTo)-[:TO_RoutesTo]->(e:APIEndpoint) RETURN r.id, r.label, e.id, e.label LIMIT 100", returns=("route_id", "route_label", "endpoint_id", "endpoint_label"), ), QueryHelperSpec( "documentation_context", "Find documentation chunks connected to code nodes.", - "MATCH (d:DocumentationChunk)-[:Documents]->(n) RETURN d.id, d.label, n.id, n.label LIMIT 50", + "MATCH (d:DocumentationChunk)-[:FROM_Documents]->(:Documents)-[:TO_Documents]->(n) RETURN d.id, d.label, n.id, n.label LIMIT 50", returns=("doc_id", "doc_label", "node_id", "node_label"), ), QueryHelperSpec( diff --git a/src/tests/test_graph_builder.py b/src/tests/test_graph_builder.py index 93a5899..6ac7a88 100644 --- a/src/tests/test_graph_builder.py +++ b/src/tests/test_graph_builder.py @@ -119,6 +119,42 @@ def test_graph_builder_uses_capture_names_as_primary_semantic_signal() -> None: assert not result.unresolved +def test_graph_builder_routes_local_imports_through_containing_scope() -> None: + parse_tree = { + "type": "Module", + "body": [ + { + "type": "ClassDef", + "name": "Loader", + "body": [ + { + "type": "FunctionDef", + "name": "load", + "body": [ + { + "type": "ImportFrom", + "module": "pathlib", + "names": [{"type": "alias", "name": "Path"}], + "line_start": 3, + }, + ], + }, + ], + }, + ], + } + + graph = GraphBuilder(default_language="python").build(parse_tree, source_path="loader.py") + + import_edge = next( + edge + for edge in graph.edges_by_type("Imports") + if graph.nodes[edge.target_id].label == "pathlib.Path" + ) + assert graph.nodes[import_edge.source_id].table == "Scope" + assert graph.nodes[import_edge.target_id].table == "ImportDeclaration" + + def test_graph_builder_emits_relation_families_advertised_by_parser_mappings() -> None: parse_tree = { "type": "Module", diff --git a/src/tests/test_materializer.py b/src/tests/test_materializer.py index 73a61bd..c4b7e0a 100644 --- a/src/tests/test_materializer.py +++ b/src/tests/test_materializer.py @@ -5,6 +5,8 @@ import pytest +import ingest.materializer as materializer_module +from db import LadybugCodeGraphStore from ingest import ( GraphMaterializer, ManifestEntry, @@ -70,11 +72,37 @@ def test_tree_sitter_python_parser_maps_sample_fixture_to_graph_tree() -> None: assert any(child["type"] == "function_definition" and child["name"] == "helper" for child in bundle.tree["children"]) +def test_scan_source_files_prunes_excluded_directories(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + source_dir = tmp_path / "src" + source_dir.mkdir() + (source_dir / "app.py").write_text("VALUE = 1\n", encoding="utf-8") + observed_dirnames: list[tuple[str, ...]] = [] + + def fake_walk(root: Path) -> object: + dirnames = [".venv", "src", "package.egg-info"] + yield Path(root).as_posix(), dirnames, [] + observed_dirnames.append(tuple(dirnames)) + for dirname in dirnames: + yield (Path(root) / dirname).as_posix(), [], ["app.py"] + + monkeypatch.setattr("ingest.materializer.os.walk", fake_walk) + materializer = GraphMaterializer(tmp_path, db_path=":memory:", manifest_path=tmp_path / "manifest.json", store=object()) + + snapshots, diagnostics = materializer._scan_source_files() + + assert observed_dirnames == [("src",)] + assert tuple(snapshots) == ("src/app.py",) + assert not diagnostics + + def test_full_materialization_writes_python_graph_to_ladybug(tmp_path: Path) -> None: pytest.importorskip("tree_sitter") pytest.importorskip("tree_sitter_python") pytest.importorskip("real_ladybug") source_root = _copy_fixture(tmp_path) + ignored_dir = source_root / ".venv" + ignored_dir.mkdir() + (ignored_dir / "ignored.py").write_text("def ignored() -> None:\n pass\n", encoding="utf-8") materializer = GraphMaterializer( source_root, @@ -95,6 +123,38 @@ def test_full_materialization_writes_python_graph_to_ladybug(tmp_path: Path) -> assert "SampleService" in _labels(materializer, "Class") assert "run" in _labels(materializer, "Method") assert {"helper", "main"} <= _labels(materializer, "Function") + assert "ignored" not in _labels(materializer, "Function") + + +def test_full_materialization_handles_local_imports_inside_methods(tmp_path: Path) -> None: + pytest.importorskip("tree_sitter") + pytest.importorskip("tree_sitter_python") + pytest.importorskip("real_ladybug") + source_root = tmp_path / "local_import_project" + source_root.mkdir() + (source_root / "service.py").write_text( + "class Loader:\n" + " def load(self) -> object:\n" + " from pathlib import Path\n" + " return Path('.')\n", + encoding="utf-8", + ) + + materializer = GraphMaterializer( + source_root, + db_path=":memory:", + manifest_path=tmp_path / "manifest.json", + include_fts=False, + ) + result = materializer.materialize(mode="full") + + assert result.rebuilt == 1 + assert "pathlib.Path" in _labels(materializer, "ImportDeclaration") + assert "Path" in _labels(materializer, "CallExpression") + metadata = materializer.store.execute( + "MATCH (n:`ImportDeclaration` {label: 'pathlib.Path'}) RETURN n.metadata" + ).get_all() + assert '"imported_name":"pathlib.Path"' in metadata[0][0] def test_changed_materialization_only_rebuilds_changed_files(tmp_path: Path) -> None: @@ -124,6 +184,104 @@ def test_changed_materialization_only_rebuilds_changed_files(tmp_path: Path) -> assert "cli.py" not in _labels(materializer, "File") +def test_full_ondisk_materialization_failure_keeps_previous_db_and_manifest( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + pytest.importorskip("tree_sitter") + pytest.importorskip("tree_sitter_python") + pytest.importorskip("real_ladybug") + source_root = tmp_path / "project" + source_root.mkdir() + service_path = source_root / "service.py" + service_path.write_text("def old_name() -> str:\n return 'old'\n", encoding="utf-8") + db_path = tmp_path / "graph.lbug" + manifest_path = tmp_path / "manifest.json" + + GraphMaterializer(source_root, db_path=db_path, manifest_path=manifest_path, include_fts=False).materialize(mode="full") + previous_manifest = manifest_path.read_text(encoding="utf-8") + service_path.write_text("def new_name() -> str:\n return 'new'\n", encoding="utf-8") + real_create_ladybug_database = materializer_module.create_ladybug_database + + def failing_create_ladybug_database(db_path: str | Path, *, include_fts: bool = True) -> LadybugCodeGraphStore: + store = real_create_ladybug_database(db_path, include_fts=include_fts) + + def fail_insert(*args: object, **kwargs: object) -> None: + raise RuntimeError("bulk insert failed") + + store.insert_graphs_bulk = fail_insert # type: ignore[method-assign] + return store + + monkeypatch.setattr(materializer_module, "create_ladybug_database", failing_create_ladybug_database) + + with pytest.raises(RuntimeError, match="bulk insert failed"): + GraphMaterializer(source_root, db_path=db_path, manifest_path=manifest_path, include_fts=False).materialize(mode="full") + + assert manifest_path.read_text(encoding="utf-8") == previous_manifest + reader = GraphMaterializer(source_root, db_path=db_path, manifest_path=manifest_path, include_fts=False) + assert "old_name" in _labels(reader, "Function") + assert "new_name" not in _labels(reader, "Function") + assert not _marker_path(manifest_path).exists() + + +def test_full_ondisk_materialization_replaces_stale_db_without_clear_graph( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + pytest.importorskip("tree_sitter") + pytest.importorskip("tree_sitter_python") + pytest.importorskip("real_ladybug") + source_root = tmp_path / "project" + source_root.mkdir() + old_path = source_root / "old_module.py" + old_path.write_text("def old_name() -> str:\n return 'old'\n", encoding="utf-8") + db_path = tmp_path / "graph.lbug" + manifest_path = tmp_path / "manifest.json" + + GraphMaterializer(source_root, db_path=db_path, manifest_path=manifest_path, include_fts=False).materialize(mode="full") + old_path.unlink() + (source_root / "new_module.py").write_text("def new_name() -> str:\n return 'new'\n", encoding="utf-8") + + def fail_clear_graph(self: LadybugCodeGraphStore) -> None: + raise AssertionError("full on-disk rebuild must not clear the target DB in place") + + monkeypatch.setattr(LadybugCodeGraphStore, "clear_graph", fail_clear_graph) + result = GraphMaterializer(source_root, db_path=db_path, manifest_path=manifest_path, include_fts=False).materialize(mode="full") + + assert result.rebuilt == 1 + reader = GraphMaterializer(source_root, db_path=db_path, manifest_path=manifest_path, include_fts=False) + assert "new_name" in _labels(reader, "Function") + assert "old_name" not in _labels(reader, "Function") + + +def test_pending_rebuild_marker_forces_changed_mode_atomic_rebuild( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + pytest.importorskip("tree_sitter") + pytest.importorskip("tree_sitter_python") + pytest.importorskip("real_ladybug") + source_root = _copy_fixture(tmp_path) + db_path = tmp_path / "graph.lbug" + manifest_path = tmp_path / "manifest.json" + + GraphMaterializer(source_root, db_path=db_path, manifest_path=manifest_path, include_fts=False).materialize(mode="full") + marker_path = _marker_path(manifest_path) + marker_path.write_text("{}\n", encoding="utf-8") + + def fail_clear_graph(self: LadybugCodeGraphStore) -> None: + raise AssertionError("marker recovery must use the atomic rebuild path") + + monkeypatch.setattr(LadybugCodeGraphStore, "clear_graph", fail_clear_graph) + result = GraphMaterializer(source_root, db_path=db_path, manifest_path=manifest_path, include_fts=False).materialize(mode="changed") + + assert result.mode == "changed" + assert result.rebuilt == 3 + assert not marker_path.exists() + reader = GraphMaterializer(source_root, db_path=db_path, manifest_path=manifest_path, include_fts=False) + assert "SampleService" in _labels(reader, "Class") + + def _entry(path: str, content_hash: str) -> ManifestEntry: return ManifestEntry( path=path, @@ -149,3 +307,7 @@ def _copy_fixture(tmp_path: Path) -> Path: def _labels(materializer: GraphMaterializer, table: str) -> set[str]: result = materializer.store.execute(f"MATCH (n:`{table}`) RETURN n.label") return {row[0] for row in result.get_all()} + + +def _marker_path(manifest_path: Path) -> Path: + return manifest_path.with_suffix(manifest_path.suffix + ".rebuild-pending") diff --git a/src/tests/test_ontology.py b/src/tests/test_ontology.py index f03073e..63f0b48 100644 --- a/src/tests/test_ontology.py +++ b/src/tests/test_ontology.py @@ -116,6 +116,23 @@ def test_query_helpers_are_read_only() -> None: assert not forbidden.search(helper.query) +def test_query_helpers_use_edge_node_relation_traversal() -> None: + direct_relation = re.compile(r"-\[:(?!FROM_|TO_)([A-Za-z][A-Za-z0-9_]*)\]->") + + for helper in QUERY_HELPERS: + assert not direct_relation.search(helper.query), helper.name + + helper_queries = {helper.name: helper.query for helper in QUERY_HELPERS} + assert "[:FROM_ResolvesTo]" in helper_queries["callgraph_neighborhood"] + assert "[:TO_ResolvesTo]" in helper_queries["callgraph_neighborhood"] + assert "[:FROM_DependsOn]" in helper_queries["dependency_map"] + assert "[:TO_DependsOn]" in helper_queries["dependency_map"] + assert "[:FROM_RoutesTo]" in helper_queries["runtime_surface"] + assert "[:TO_RoutesTo]" in helper_queries["runtime_surface"] + assert "[:FROM_Documents]" in helper_queries["documentation_context"] + assert "[:TO_Documents]" in helper_queries["documentation_context"] + + def test_lookup_helpers_return_expected_specs() -> None: assert get_node_type("Class").name == "Class" assert get_relation_type("Calls").name == "Calls" diff --git a/src/tests/test_schema.py b/src/tests/test_schema.py index 95eb88d..6f7cf92 100644 --- a/src/tests/test_schema.py +++ b/src/tests/test_schema.py @@ -2,6 +2,7 @@ import pytest +from core import CodeGraph, GraphEdge, GraphNode from db import ( LadybugCodeGraphStore, build_ladybug_schema, @@ -117,3 +118,28 @@ def test_ladybug_store_schema_setup_is_idempotent() -> None: store = create_ladybug_database(":memory:") store.ensure_schema() + + +def test_ladybug_store_bulk_loader_groups_rows_by_table() -> None: + graph = CodeGraph() + graph.add_node(GraphNode(id="file:service", table="File", label="service.py", kind="source_file")) + graph.add_node(GraphNode(id="function:one", table="Function", label="one", kind="function")) + graph.add_node(GraphNode(id="function:two", table="Function", label="two", kind="function")) + graph.add_edge(GraphEdge(id="contains:one", type="Contains", source_id="file:service", target_id="function:one")) + graph.add_edge(GraphEdge(id="contains:two", type="Contains", source_id="file:service", target_id="function:two")) + store = object.__new__(LadybugCodeGraphStore) + statements: list[str] = [] + store.execute = lambda statement, parameters=None: statements.append(statement) # type: ignore[method-assign] + + stats = store.insert_graphs_bulk([graph]) + + assert stats.node_rows == 3 + assert stats.edge_rows == 2 + assert stats.connector_rows == 4 + assert stats.copy_calls == 5 + assert len(statements) == 5 + assert any(statement.startswith("COPY `File`") and statement.endswith('";') for statement in statements) + assert any(statement.startswith("COPY `Function`") and statement.endswith('";') for statement in statements) + assert any(statement.startswith("COPY `Contains`") and statement.endswith('";') for statement in statements) + assert any('COPY `FROM_Contains`' in statement and 'from="File", to="Contains"' in statement for statement in statements) + assert any('COPY `TO_Contains`' in statement and 'from="Contains", to="Function"' in statement for statement in statements) From 90056f7e18e97ccfa0fa146575803a4750c60f99 Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Mon, 25 May 2026 10:26:36 +0930 Subject: [PATCH 10/53] Feat: Add comprehensive workflow - Introduced tests for the MaterializationManifest and GraphMaterializer to validate file changes and materialization processes. - Implemented tests for ontology structure and relationships, ensuring all required node types and relation endpoints are declared. - Created tests for schema generation, verifying that all ontology nodes and edge nodes are correctly declared and that FTS indexes are created as expected. - Developed search functionality tests to ensure ranking and context retrieval work as intended, including handling of malicious queries. - Added setup workflow tests to validate CLI behavior, configuration creation, and error handling during setup. --- README.md | 105 +++++- pyproject.toml | 10 +- src/codebase_graph/__init__.py | 3 + src/{ => codebase_graph}/cli/__init__.py | 52 ++- src/{ => codebase_graph}/core/__init__.py | 0 src/{ => codebase_graph}/core/graph.py | 2 +- src/{ => codebase_graph}/db/__init__.py | 0 src/{ => codebase_graph}/db/schema.py | 2 +- src/{ => codebase_graph}/db/store.py | 15 +- src/{ => codebase_graph}/extract/__init__.py | 0 .../extract/graph_builder.py | 14 +- src/{ => codebase_graph}/ingest/__init__.py | 2 + src/codebase_graph/ingest/document_parser.py | 104 +++++ .../ingest/materializer.py | 13 +- .../ingest/tree_sitter_parser.py | 7 +- src/codebase_graph/mcp/__init__.py | 5 + src/codebase_graph/mcp/server.py | 354 ++++++++++++++++++ src/{ => codebase_graph}/memory/__init__.py | 0 src/{ => codebase_graph}/ontology/__init__.py | 0 src/{ => codebase_graph}/ontology/ontology.py | 0 .../reasoning/__init__.py | 0 .../reasoning/context_builder.py | 4 +- .../retrieval/__init__.py | 0 src/{ => codebase_graph}/retrieval/search.py | 4 +- src/codebase_graph/setup/__init__.py | 24 ++ src/codebase_graph/setup/instructions.py | 95 +++++ src/codebase_graph/setup/mcp_config.py | 92 +++++ src/codebase_graph/setup/orchestrator.py | 106 ++++++ src/codebase_graph/setup/preflight.py | 21 ++ src/codebase_graph/setup/state.py | 128 +++++++ src/tests/__init__.py | 1 - {src/tests => tests}/test_graph_builder.py | 4 +- {src/tests => tests}/test_materializer.py | 21 +- {src/tests => tests}/test_ontology.py | 2 +- {src/tests => tests}/test_schema.py | 6 +- {src/tests => tests}/test_search.py | 8 +- tests/test_setup_workflow.py | 192 ++++++++++ 37 files changed, 1347 insertions(+), 49 deletions(-) create mode 100644 src/codebase_graph/__init__.py rename src/{ => codebase_graph}/cli/__init__.py (60%) rename src/{ => codebase_graph}/core/__init__.py (100%) rename src/{ => codebase_graph}/core/graph.py (98%) rename src/{ => codebase_graph}/db/__init__.py (100%) rename src/{ => codebase_graph}/db/schema.py (97%) rename src/{ => codebase_graph}/db/store.py (96%) rename src/{ => codebase_graph}/extract/__init__.py (100%) rename src/{ => codebase_graph}/extract/graph_builder.py (99%) rename src/{ => codebase_graph}/ingest/__init__.py (88%) create mode 100644 src/codebase_graph/ingest/document_parser.py rename src/{ => codebase_graph}/ingest/materializer.py (98%) rename src/{ => codebase_graph}/ingest/tree_sitter_parser.py (97%) create mode 100644 src/codebase_graph/mcp/__init__.py create mode 100644 src/codebase_graph/mcp/server.py rename src/{ => codebase_graph}/memory/__init__.py (100%) rename src/{ => codebase_graph}/ontology/__init__.py (100%) rename src/{ => codebase_graph}/ontology/ontology.py (100%) rename src/{ => codebase_graph}/reasoning/__init__.py (100%) rename src/{ => codebase_graph}/reasoning/context_builder.py (98%) rename src/{ => codebase_graph}/retrieval/__init__.py (100%) rename src/{ => codebase_graph}/retrieval/search.py (98%) create mode 100644 src/codebase_graph/setup/__init__.py create mode 100644 src/codebase_graph/setup/instructions.py create mode 100644 src/codebase_graph/setup/mcp_config.py create mode 100644 src/codebase_graph/setup/orchestrator.py create mode 100644 src/codebase_graph/setup/preflight.py create mode 100644 src/codebase_graph/setup/state.py delete mode 100644 src/tests/__init__.py rename {src/tests => tests}/test_graph_builder.py (98%) rename {src/tests => tests}/test_materializer.py (95%) rename {src/tests => tests}/test_ontology.py (99%) rename {src/tests => tests}/test_schema.py (97%) rename {src/tests => tests}/test_search.py (97%) create mode 100644 tests/test_setup_workflow.py diff --git a/README.md b/README.md index 6be8c38..061467b 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,110 @@ # codebaseGraph -`codebaseGraph` is a generic project/code knowledge graph engine for coding repositories. The current filesystem materializer scans Python `.py` files, builds a typed graph of files, modules, symbols, imports, calls, dependencies, and entry points, and exposes search, compact context, schema, and read-only query helpers. The ontology and graph builder can represent documentation chunks from normalized parser or capture input, but Markdown and other documentation files are not scanned by the materializer yet. +`codebaseGraph` is a generic project/code knowledge graph engine for coding repositories. It materializes Python source, `AGENTS.md`, `CLAUDE.md`, Markdown, and MDX files into a LadyBugDB-backed graph, then exposes search, compact context, schema, query helpers, and a read-only MCP tool surface for coding agents. -## Install for local development +LadyBugDB is a required runtime dependency. A normal production install must include `real_ladybug`; setup fails before creating repository state if the runtime cannot open a graph database. + +## Production install + +```bash +python -m pip install codebase-graph +``` + +From a repository root, run: + +```bash +codebase-graph setup --repo-root . +``` + +Setup creates: + +```text +.codebaseGraph/ + config.json + manifest.json + _graph.ldb +``` + +For a repository named `my-service`, the database path is exactly `.codebaseGraph/my-service_graph.ldb`. + +The setup command also: + +- Materializes the repository graph into the repo-local database. +- Writes or updates one marked codebaseGraph block in `AGENTS.md` or `CLAUDE.md`. +- Writes a Codex or Claude-compatible MCP JSON config entry named `codebaseGraph`, unless skipped. + +Useful options: + +```bash +codebase-graph setup --repo-root /path/to/repo +codebase-graph setup --mcp-client claude +codebase-graph setup --mcp-config-path /tmp/mcp.json +codebase-graph setup --dry-run +codebase-graph setup --skip-mcp-config +codebase-graph setup --instructions-target claude +``` + +`--dry-run` returns the MCP config patch without writing the MCP client file. Repository graph state and instruction handling still run so the graph can be verified. + +## MCP usage + +Setup writes an MCP server entry equivalent to: + +```json +{ + "mcpServers": { + "codebaseGraph": { + "command": "codebase-graph", + "args": ["mcp", "serve", "--config", ".codebaseGraph/config.json"] + } + } +} +``` + +The server can also be run directly: + +```bash +codebase-graph mcp serve --config .codebaseGraph/config.json +codebase-graph-mcp --config .codebaseGraph/config.json +``` + +Available MCP tools: + +- `graph_health` +- `graph_search` +- `graph_context` +- `graph_schema` +- `graph_query_helpers` +- `graph_query` with write-like statements blocked + +## CLI search + +The legacy materializer/search commands are still available. Setup reports the explicit database and manifest paths to use with them: + +```bash +codebase-graph search SampleService \ + --source-root . \ + --db .codebaseGraph/_graph.ldb \ + --manifest .codebaseGraph/manifest.json +``` + +## Development install ```bash python -m pip install -e .[dev] ``` + +Run checks: + +```bash +python -m pytest +ruff check . +``` + +## Troubleshooting + +- Missing LadyBugDB: install a package build that includes `real_ladybug`; setup will fail before creating `.codebaseGraph`. +- Stale graph: rerun `codebase-graph setup --repo-root .` after material source or documentation changes. +- Broken MCP config: rerun setup with `--mcp-config-path` pointing at the client JSON file, or use `--dry-run` to inspect the server block. +- Unsupported files: binary, vendor, cache, virtualenv, build, dist, `.codebase_graph`, and `.codebaseGraph` paths are skipped. +- Lock/contention errors: stop other graph materialization or MCP processes using the same `.codebaseGraph/_graph.ldb`, then rerun setup. diff --git a/pyproject.toml b/pyproject.toml index d42f4db..6b1c58f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,6 +10,7 @@ readme = "README.md" requires-python = ">=3.10" authors = [{ name = "Rabii Chaarani" }] dependencies = [ + "real_ladybug", "tomli; python_version < '3.11'", "tree-sitter", "tree-sitter-python", @@ -23,15 +24,18 @@ classifiers = [ ] [project.optional-dependencies] -ladybug = ["real_ladybug"] +ladybug = [] parquet = ["pyarrow"] dev = ["pytest", "ruff"] [project.scripts] -codebase-graph = "cli:main" +codebase-graph = "codebase_graph.cli:main" +codebase-graph-mcp = "codebase_graph.mcp.server:main" [tool.setuptools.packages.find] where = ["src"] +include = ["codebase_graph*"] +exclude = ["tests*", "codebase_graph.egg-info*"] [tool.ruff] line-length = 120 @@ -39,4 +43,4 @@ target-version = "py310" [tool.pytest.ini_options] pythonpath = ["src"] -testpaths = ["src/tests"] +testpaths = ["tests"] diff --git a/src/codebase_graph/__init__.py b/src/codebase_graph/__init__.py new file mode 100644 index 0000000..0edc212 --- /dev/null +++ b/src/codebase_graph/__init__.py @@ -0,0 +1,3 @@ +"""Production namespace for codebase-graph.""" + +__all__ = [] diff --git a/src/cli/__init__.py b/src/codebase_graph/cli/__init__.py similarity index 60% rename from src/cli/__init__.py rename to src/codebase_graph/cli/__init__.py index 479b03d..d0b88e3 100644 --- a/src/cli/__init__.py +++ b/src/codebase_graph/cli/__init__.py @@ -5,9 +5,10 @@ from collections.abc import Sequence from pathlib import Path -from ingest import GraphMaterializer -from ontology import CONTEXT_PROFILES -from retrieval import SearchRequest, SearchService +from codebase_graph.ingest import GraphMaterializer +from codebase_graph.ontology import CONTEXT_PROFILES +from codebase_graph.retrieval import SearchRequest, SearchService +from codebase_graph.setup import SetupError, SetupOptions, run_setup def main(argv: Sequence[str] | None = None) -> int: @@ -27,6 +28,29 @@ def main(argv: Sequence[str] | None = None) -> int: context_parser = subparsers.add_parser("context", help="Return compact context for a search query") _add_search_arguments(context_parser) + setup_parser = subparsers.add_parser("setup", help="Bootstrap codebaseGraph state for a repository") + setup_parser.add_argument("--repo-root", default=".", help="Repository root to configure") + setup_parser.add_argument("--mcp-client", choices=("codex", "claude", "none"), default="codex") + setup_parser.add_argument("--mcp-config-path", default=None, help="Override MCP JSON config path") + setup_parser.add_argument("--skip-mcp-config", action="store_true", help="Do not write MCP client config") + setup_parser.add_argument("--dry-run", action="store_true", help="Return the MCP config patch without writing it") + setup_parser.add_argument( + "--instructions-target", + choices=("auto", "agents", "claude", "skip"), + default="auto", + help="Instruction file to update", + ) + setup_parser.add_argument("--mode", choices=("full", "changed"), default="changed", help="Materialization mode") + setup_parser.add_argument("--json", action="store_true", help="Emit JSON output") + + mcp_parser = subparsers.add_parser("mcp", help="Run or inspect the MCP server") + mcp_subparsers = mcp_parser.add_subparsers(dest="mcp_command", required=True) + serve_parser = mcp_subparsers.add_parser("serve", help="Serve graph tools over MCP stdio") + serve_parser.add_argument("--repo-root", default=".", help="Repository root containing .codebaseGraph/config.json") + serve_parser.add_argument("--config", default=None, help="Path to .codebaseGraph/config.json") + serve_parser.add_argument("--db", default=None, help="Override LadyBugDB path") + serve_parser.add_argument("--manifest", default=None, help="Override manifest path") + args = parser.parse_args(argv) if args.command == "materialize": materializer = GraphMaterializer( @@ -61,6 +85,28 @@ def main(argv: Sequence[str] | None = None) -> int: payload = SearchService(materializer.store).search(request) print(json.dumps(payload.as_dict(), indent=2, sort_keys=True)) return 0 + if args.command == "setup": + try: + result = run_setup( + SetupOptions( + repo_root=args.repo_root, + mcp_client=args.mcp_client, + mcp_config_path=args.mcp_config_path, + skip_mcp_config=args.skip_mcp_config, + dry_run=args.dry_run, + instructions_target=args.instructions_target, + mode=args.mode, + ) + ) + except SetupError as exc: + parser.error(str(exc)) + print(json.dumps(result.as_dict(), indent=2, sort_keys=True)) + return 0 + if args.command == "mcp" and args.mcp_command == "serve": + from codebase_graph.mcp.server import serve_stdio + + serve_stdio(repo_root=args.repo_root, config_path=args.config, db_path=args.db, manifest_path=args.manifest) + return 0 parser.error(f"Unknown command: {args.command}") return 2 diff --git a/src/core/__init__.py b/src/codebase_graph/core/__init__.py similarity index 100% rename from src/core/__init__.py rename to src/codebase_graph/core/__init__.py diff --git a/src/core/graph.py b/src/codebase_graph/core/graph.py similarity index 98% rename from src/core/graph.py rename to src/codebase_graph/core/graph.py index 3467e18..c469bae 100644 --- a/src/core/graph.py +++ b/src/codebase_graph/core/graph.py @@ -3,7 +3,7 @@ from dataclasses import dataclass, field from typing import Any -from ontology import ONTOLOGY_NAME, get_relation_type +from codebase_graph.ontology import ONTOLOGY_NAME, get_relation_type @dataclass(slots=True) diff --git a/src/db/__init__.py b/src/codebase_graph/db/__init__.py similarity index 100% rename from src/db/__init__.py rename to src/codebase_graph/db/__init__.py diff --git a/src/db/schema.py b/src/codebase_graph/db/schema.py similarity index 97% rename from src/db/schema.py rename to src/codebase_graph/db/schema.py index 9665d99..2b8fa8b 100644 --- a/src/db/schema.py +++ b/src/codebase_graph/db/schema.py @@ -2,7 +2,7 @@ from collections.abc import Iterable -from ontology import EDGE_FIELDS, NODE_TYPES, RELATION_TYPES, SEARCH_INDEXES, FieldSpec +from codebase_graph.ontology import EDGE_FIELDS, NODE_TYPES, RELATION_TYPES, SEARCH_INDEXES, FieldSpec TYPE_MAP = { "string": "STRING", diff --git a/src/db/store.py b/src/codebase_graph/db/store.py similarity index 96% rename from src/db/store.py rename to src/codebase_graph/db/store.py index 651af13..7d88c2d 100644 --- a/src/db/store.py +++ b/src/codebase_graph/db/store.py @@ -9,8 +9,8 @@ from pathlib import Path from typing import Any -from core import CodeGraph -from ontology import NODE_TYPES, RELATION_TYPES +from codebase_graph.core import CodeGraph +from codebase_graph.ontology import NODE_TYPES, RELATION_TYPES from .schema import build_ladybug_schema, build_ladybug_schema_statements, quote_identifier @@ -35,7 +35,8 @@ def __init__(self, db_path: str | Path = ":memory:", *, include_fts: bool = True import real_ladybug as lb except ImportError as exc: raise LadybugUnavailableError( - "LadyBugDB Python bindings are not installed. Install `real_ladybug` or `codebase-graph[ladybug]`." + "LadyBugDB Python bindings are required for codebaseGraph. " + "Install a valid `codebase-graph` runtime with `real_ladybug` available." ) from exc self._lb = lb @@ -61,6 +62,12 @@ def close(self) -> None: self.conn.close() self.db.close() + def __enter__(self) -> LadybugCodeGraphStore: + return self + + def __exit__(self, exc_type: object, exc: object, traceback: object) -> None: + self.close() + def clear_graph(self) -> None: for relation_type in RELATION_TYPES: self._execute_ignoring_missing(f"MATCH ()-[r:{quote_identifier(f'FROM_{relation_type.name}')}]->() DELETE r") @@ -149,7 +156,7 @@ def delete_partition( self._delete_node(node_id, node_type) def read_manifest(self, path: str | Path) -> Any: - from ingest.materializer import MaterializationManifest + from codebase_graph.ingest.materializer import MaterializationManifest return MaterializationManifest.load(Path(path)) diff --git a/src/extract/__init__.py b/src/codebase_graph/extract/__init__.py similarity index 100% rename from src/extract/__init__.py rename to src/codebase_graph/extract/__init__.py diff --git a/src/extract/graph_builder.py b/src/codebase_graph/extract/graph_builder.py similarity index 99% rename from src/extract/graph_builder.py rename to src/codebase_graph/extract/graph_builder.py index 749cad7..0258e95 100644 --- a/src/extract/graph_builder.py +++ b/src/codebase_graph/extract/graph_builder.py @@ -6,8 +6,8 @@ from pathlib import Path from typing import Any -from core import CodeGraph, GraphEdge, GraphNode -from ontology import ONTOLOGY_NAME, get_relation_type, node_type_names, relation_type_names +from codebase_graph.core import CodeGraph, GraphEdge, GraphNode +from codebase_graph.ontology import ONTOLOGY_NAME, get_relation_type, node_type_names, relation_type_names @dataclass(frozen=True, slots=True) @@ -622,7 +622,7 @@ def _semantic_node( byte_end=parser_node.byte_end, tree_sitter_node_type=parser_node.node_type, capture_name=parser_node.capture_name, - summary=semantic_label, + summary=_summary_for(table, semantic_label, parser_node), metadata={"canonical_key": stable_key, **(metadata or {})}, ) added = self._graph.add_node(node) @@ -985,6 +985,8 @@ def _table_from_capture(capture_name: str, owner: ScopeFrame) -> str | None: return "APIEndpoint" if capture == "route": return "Route" + if capture == "doc.source": + return "DocumentationSource" if capture.startswith("doc"): return "DocumentationChunk" if capture in {"literal", "string", "number"}: @@ -1062,6 +1064,12 @@ def _label_for(node: ParserNode) -> str: return node.text.strip() or node.node_type +def _summary_for(table: str, label: str, node: ParserNode) -> str: + if table in {"DocumentationSource", "DocumentationChunk"} and node.text.strip(): + return node.text.strip() + return label + + def _value_label(value: Any) -> str: if value is None: return "" diff --git a/src/ingest/__init__.py b/src/codebase_graph/ingest/__init__.py similarity index 88% rename from src/ingest/__init__.py rename to src/codebase_graph/ingest/__init__.py index 64b2fad..9352568 100644 --- a/src/ingest/__init__.py +++ b/src/codebase_graph/ingest/__init__.py @@ -10,9 +10,11 @@ SourceSnapshot, ) from .tree_sitter_parser import ParserUnavailableError, TreeSitterPythonParser, parser_for_language +from .document_parser import MarkdownDocumentParser __all__ = [ "GraphMaterializer", + "MarkdownDocumentParser", "ManifestDiff", "ManifestEntry", "MaterializationManifest", diff --git a/src/codebase_graph/ingest/document_parser.py b/src/codebase_graph/ingest/document_parser.py new file mode 100644 index 0000000..5c75e4c --- /dev/null +++ b/src/codebase_graph/ingest/document_parser.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from codebase_graph.extract import ParseBundle + +HEADING_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*$") + + +@dataclass(frozen=True, slots=True) +class MarkdownDocumentParser: + language: str = "markdown" + parser_version: str = "markdown-docs-v1" + + def parse_file( + self, + path: Path, + *, + relative_path: str, + source_root: Path, + repository_label: str, + content_hash: str, + ) -> ParseBundle: + source_text = path.read_text(encoding="utf-8") + return ParseBundle( + language=self.language, + path=relative_path, + source_text=source_text, + captures=_document_captures(relative_path, source_text), + repository_label=repository_label, + source_root=source_root.as_posix(), + content_hash=content_hash, + ) + + +def _document_captures(path: str, source_text: str) -> tuple[dict[str, Any], ...]: + lines = source_text.splitlines() + total_lines = max(len(lines), 1) + captures: list[dict[str, Any]] = [ + { + "capture_name": "doc.source", + "node": { + "type": "DocumentationSource", + "name": path, + "line_start": 1, + "line_end": total_lines, + "text": _summary(source_text), + }, + } + ] + for index, section in enumerate(_sections(lines), start=1): + label = section.heading or f"{path} section {index}" + captures.append( + { + "capture_name": "doc.chunk", + "node": { + "type": "DocumentationChunk", + "name": label, + "heading": section.heading, + "level": section.level, + "line_start": section.line_start, + "line_end": section.line_end, + "text": _summary(section.text), + }, + } + ) + return tuple(captures) + + +@dataclass(frozen=True, slots=True) +class _Section: + heading: str + level: int + line_start: int + line_end: int + text: str + + +def _sections(lines: list[str]) -> tuple[_Section, ...]: + headings: list[tuple[int, int, str]] = [] + for line_number, line in enumerate(lines, start=1): + match = HEADING_RE.match(line) + if match is None: + continue + headings.append((line_number, len(match.group(1)), match.group(2).strip())) + + if not headings: + text = "\n".join(lines).strip() + return (_Section("", 0, 1, max(len(lines), 1), text),) if text else () + + sections: list[_Section] = [] + for index, (line_start, level, heading) in enumerate(headings): + line_end = headings[index + 1][0] - 1 if index + 1 < len(headings) else len(lines) + text = "\n".join(lines[line_start - 1 : line_end]).strip() + if text: + sections.append(_Section(heading, level, line_start, line_end, text)) + return tuple(sections) + + +def _summary(text: str) -> str: + return text.strip()[:2000] diff --git a/src/ingest/materializer.py b/src/codebase_graph/ingest/materializer.py similarity index 98% rename from src/ingest/materializer.py rename to src/codebase_graph/ingest/materializer.py index 95f4dc2..4af3973 100644 --- a/src/ingest/materializer.py +++ b/src/codebase_graph/ingest/materializer.py @@ -10,10 +10,10 @@ from pathlib import Path from typing import Any, Literal -from core import CodeGraph -from db import LadybugCodeGraphStore, create_ladybug_database -from extract import GraphBuilder -from ontology import ONTOLOGY_NAME +from codebase_graph.core import CodeGraph +from codebase_graph.db import LadybugCodeGraphStore, create_ladybug_database +from codebase_graph.extract import GraphBuilder +from codebase_graph.ontology import ONTOLOGY_NAME from .tree_sitter_parser import ParserUnavailableError, parser_for_language @@ -23,8 +23,8 @@ DEFAULT_STATE_DIR = ".codebase_graph" DEFAULT_MANIFEST_NAME = "manifest.json" DEFAULT_DB_NAME = "graph.lbug" -PARSER_VERSION = "tree-sitter-python-v1" -SUPPORTED_SUFFIXES = {".py": "python"} +PARSER_VERSION = "tree-sitter-python-v1+markdown-docs-v1" +SUPPORTED_SUFFIXES = {".py": "python", ".md": "markdown", ".mdx": "markdown"} EXCLUDED_PARTS = { ".git", ".venv", @@ -34,6 +34,7 @@ "build", "dist", ".codebase_graph", + ".codebaseGraph", } diff --git a/src/ingest/tree_sitter_parser.py b/src/codebase_graph/ingest/tree_sitter_parser.py similarity index 97% rename from src/ingest/tree_sitter_parser.py rename to src/codebase_graph/ingest/tree_sitter_parser.py index 21f307a..27873dd 100644 --- a/src/ingest/tree_sitter_parser.py +++ b/src/codebase_graph/ingest/tree_sitter_parser.py @@ -5,7 +5,8 @@ from pathlib import Path from typing import Any -from extract import ParseBundle +from codebase_graph.extract import ParseBundle +from .document_parser import MarkdownDocumentParser class ParserUnavailableError(RuntimeError): @@ -44,9 +45,11 @@ def parse_source(self, source_text: str) -> dict[str, Any]: return _convert_node(tree.root_node, source_bytes) -def parser_for_language(language: str) -> TreeSitterPythonParser: +def parser_for_language(language: str) -> TreeSitterPythonParser | MarkdownDocumentParser: if language == "python": return TreeSitterPythonParser() + if language == "markdown": + return MarkdownDocumentParser() raise ValueError(f"Unsupported materializer language: {language}") diff --git a/src/codebase_graph/mcp/__init__.py b/src/codebase_graph/mcp/__init__.py new file mode 100644 index 0000000..fe546b1 --- /dev/null +++ b/src/codebase_graph/mcp/__init__.py @@ -0,0 +1,5 @@ +"""MCP server surface for codebaseGraph.""" + +from .server import McpGraphServer, handle_tool_call, serve_stdio + +__all__ = ["McpGraphServer", "handle_tool_call", "serve_stdio"] diff --git a/src/codebase_graph/mcp/server.py b/src/codebase_graph/mcp/server.py new file mode 100644 index 0000000..eccc1dd --- /dev/null +++ b/src/codebase_graph/mcp/server.py @@ -0,0 +1,354 @@ +from __future__ import annotations + +import json +import re +import sys +from dataclasses import dataclass +from importlib.metadata import PackageNotFoundError, version +from pathlib import Path +from typing import Any + +from codebase_graph.db import LadybugCodeGraphStore, create_ladybug_database +from codebase_graph.ontology import QUERY_HELPERS, schema_payload +from codebase_graph.reasoning import CompactContextBuilder +from codebase_graph.retrieval import SearchRequest, SearchService +from codebase_graph.setup.state import derive_setup_paths, load_setup_config + +READ_ONLY_DENY_RE = re.compile( + r"\b(CREATE|DELETE|SET|MERGE|DROP|COPY|INSERT|LOAD|INSTALL|DETACH|REMOVE|ALTER|RENAME)\b", + re.IGNORECASE, +) + + +@dataclass(frozen=True, slots=True) +class GraphRuntimeConfig: + repo_root: Path + db_path: Path + manifest_path: Path | None = None + + +class McpGraphServer: + def __init__(self, runtime: GraphRuntimeConfig) -> None: + self.runtime = runtime + + @classmethod + def from_paths( + cls, + *, + repo_root: str | Path = ".", + config_path: str | Path | None = None, + db_path: str | Path | None = None, + manifest_path: str | Path | None = None, + ) -> McpGraphServer: + runtime = _runtime_config( + repo_root=repo_root, + config_path=config_path, + db_path=db_path, + manifest_path=manifest_path, + ) + return cls(runtime) + + def handle_json_rpc(self, message: dict[str, Any]) -> dict[str, Any] | None: + method = str(message.get("method", "")) + request_id = message.get("id") + if method.startswith("notifications/"): + return None + try: + if method == "initialize": + result = { + "protocolVersion": "2024-11-05", + "capabilities": {"tools": {}}, + "serverInfo": {"name": "codebaseGraph", "version": _package_version()}, + } + elif method == "ping": + result = {} + elif method == "tools/list": + result = {"tools": _tool_specs()} + elif method == "tools/call": + params = dict(message.get("params") or {}) + payload = handle_tool_call( + str(params.get("name", "")), + dict(params.get("arguments") or {}), + runtime=self.runtime, + ) + result = _tool_result(payload) + else: + return _rpc_error(request_id, -32601, f"Unsupported MCP method: {method}") + except Exception as exc: + return _rpc_error(request_id, -32000, str(exc)) + return {"jsonrpc": "2.0", "id": request_id, "result": result} + + +def handle_tool_call(name: str, arguments: dict[str, Any], *, runtime: GraphRuntimeConfig) -> dict[str, Any]: + if name == "graph_health": + return _health(runtime) + if name == "graph_schema": + return schema_payload() + if name == "graph_query_helpers": + return {"query_helpers": [helper.as_dict() for helper in QUERY_HELPERS]} + if name == "graph_search": + with _store(runtime) as store: + request = _search_request(arguments) + return SearchService(store).search(request).as_dict() + if name == "graph_context": + with _store(runtime) as store: + return _context_payload(store, arguments) + if name == "graph_query": + with _store(runtime) as store: + return _query_payload(store, arguments) + raise ValueError(f"Unknown codebaseGraph MCP tool: {name}") + + +def serve_stdio( + *, + repo_root: str | Path = ".", + config_path: str | Path | None = None, + db_path: str | Path | None = None, + manifest_path: str | Path | None = None, +) -> None: + server = McpGraphServer.from_paths( + repo_root=repo_root, + config_path=config_path, + db_path=db_path, + manifest_path=manifest_path, + ) + while True: + message = _read_message(sys.stdin.buffer) + if message is None: + return + response = server.handle_json_rpc(message) + if response is not None: + _write_message(sys.stdout.buffer, response) + + +def main() -> int: + import argparse + + parser = argparse.ArgumentParser(prog="codebase-graph-mcp") + parser.add_argument("--repo-root", default=".", help="Repository root containing .codebaseGraph/config.json") + parser.add_argument("--config", default=None, help="Path to .codebaseGraph/config.json") + parser.add_argument("--db", default=None, help="Override LadyBugDB path") + parser.add_argument("--manifest", default=None, help="Override manifest path") + args = parser.parse_args() + serve_stdio(repo_root=args.repo_root, config_path=args.config, db_path=args.db, manifest_path=args.manifest) + return 0 + + +def _runtime_config( + *, + repo_root: str | Path, + config_path: str | Path | None, + db_path: str | Path | None, + manifest_path: str | Path | None, +) -> GraphRuntimeConfig: + root = Path(repo_root).expanduser().resolve() + config = Path(config_path).expanduser().resolve() if config_path else derive_setup_paths(root).config_path + payload: dict[str, Any] = {} + if config.exists(): + payload = load_setup_config(config) + elif db_path is None: + raise FileNotFoundError(f"codebaseGraph setup config is missing: {config}") + resolved_db = Path(db_path or payload["database_path"]).expanduser().resolve() + resolved_manifest = Path(manifest_path or payload.get("manifest_path", "")).expanduser().resolve() if (manifest_path or payload.get("manifest_path")) else None + if not resolved_db.exists(): + raise FileNotFoundError(f"codebaseGraph database is missing: {resolved_db}") + return GraphRuntimeConfig(repo_root=root, db_path=resolved_db, manifest_path=resolved_manifest) + + +def _store(runtime: GraphRuntimeConfig) -> LadybugCodeGraphStore: + return create_ladybug_database(runtime.db_path, include_fts=True) + + +def _health(runtime: GraphRuntimeConfig) -> dict[str, Any]: + return { + "ok": runtime.db_path.exists(), + "repo_root": runtime.repo_root.as_posix(), + "database_path": runtime.db_path.as_posix(), + "manifest_path": runtime.manifest_path.as_posix() if runtime.manifest_path else None, + } + + +def _search_request(arguments: dict[str, Any]) -> SearchRequest: + request = SearchRequest( + query=str(arguments.get("query", "")), + limit=int(arguments.get("limit", 3)), + profile=str(arguments.get("profile", "brief")), + budget=int(arguments.get("budget", 600)), + max_depth=_optional_int(arguments.get("max_depth")), + ) + request.validate() + return request + + +def _context_payload(store: LadybugCodeGraphStore, arguments: dict[str, Any]) -> dict[str, Any]: + node_id = str(arguments.get("node_id") or "") + node_type = str(arguments.get("node_type") or "") + if node_id and node_type: + profile = str(arguments.get("profile", "brief")) + context = CompactContextBuilder(store).build( + node_id, + node_type, + profile=profile, + limit=int(arguments.get("limit", 3)), + budget=int(arguments.get("budget", 600)), + max_depth=_optional_int(arguments.get("max_depth")), + ) + return { + "node_id": node_id, + "node_type": node_type, + "profile": profile, + "context": [node.as_dict() for node in context], + } + return SearchService(store).search(_search_request(arguments)).as_dict() + + +def _query_payload(store: LadybugCodeGraphStore, arguments: dict[str, Any]) -> dict[str, Any]: + statement = str(arguments.get("statement") or arguments.get("query") or "").strip() + if not statement: + raise ValueError("graph_query requires a non-empty statement") + _validate_read_only_statement(statement) + parameters = arguments.get("parameters") or {} + if not isinstance(parameters, dict): + raise ValueError("graph_query parameters must be a JSON object") + limit = int(arguments.get("limit", 100)) + rows = store.execute(statement, parameters).get_all() + return { + "statement": statement, + "row_count": len(rows), + "rows": [_row_values(row) for row in rows[:limit]], + "truncated": len(rows) > limit, + } + + +def _validate_read_only_statement(statement: str) -> None: + stripped = statement.strip().rstrip(";") + if ";" in stripped: + raise ValueError("graph_query accepts one read-only statement at a time") + match = READ_ONLY_DENY_RE.search(stripped) + if match is not None: + raise ValueError(f"graph_query is read-only; blocked keyword: {match.group(1).upper()}") + + +def _row_values(row: Any) -> list[Any]: + try: + return [_json_safe(value) for value in row] + except TypeError: + return [_json_safe(row)] + + +def _json_safe(value: Any) -> Any: + if value is None or isinstance(value, (str, int, float, bool)): + return value + if isinstance(value, (list, tuple)): + return [_json_safe(item) for item in value] + if isinstance(value, dict): + return {str(key): _json_safe(item) for key, item in value.items()} + return str(value) + + +def _tool_result(payload: dict[str, Any]) -> dict[str, Any]: + return { + "content": [{"type": "text", "text": json.dumps(payload, indent=2, sort_keys=True)}], + "structuredContent": payload, + "isError": False, + } + + +def _tool_specs() -> list[dict[str, Any]]: + return [ + { + "name": "graph_health", + "description": "Check the configured codebaseGraph database path and manifest path.", + "inputSchema": {"type": "object", "properties": {}, "additionalProperties": False}, + }, + { + "name": "graph_search", + "description": "Search code, documentation, paths, and dependencies with compact graph context.", + "inputSchema": _search_schema(required=("query",)), + }, + { + "name": "graph_context", + "description": "Return compact context for a search query or explicit node_id/node_type pair.", + "inputSchema": _search_schema(required=()), + }, + { + "name": "graph_schema", + "description": "Return ontology schema, search indexes, context profiles, and query helper metadata.", + "inputSchema": {"type": "object", "properties": {}, "additionalProperties": False}, + }, + { + "name": "graph_query_helpers", + "description": "Return named read-only query helpers for common graph exploration tasks.", + "inputSchema": {"type": "object", "properties": {}, "additionalProperties": False}, + }, + { + "name": "graph_query", + "description": "Execute a restricted read-only graph query against the configured database.", + "inputSchema": { + "type": "object", + "properties": { + "statement": {"type": "string"}, + "parameters": {"type": "object"}, + "limit": {"type": "integer", "minimum": 1}, + }, + "required": ["statement"], + "additionalProperties": False, + }, + }, + ] + + +def _search_schema(*, required: tuple[str, ...]) -> dict[str, Any]: + return { + "type": "object", + "properties": { + "query": {"type": "string"}, + "limit": {"type": "integer", "minimum": 1}, + "profile": {"type": "string"}, + "budget": {"type": "integer", "minimum": 0}, + "max_depth": {"type": "integer", "minimum": 0}, + "node_id": {"type": "string"}, + "node_type": {"type": "string"}, + }, + "required": list(required), + "additionalProperties": False, + } + + +def _optional_int(value: Any) -> int | None: + if value is None or value == "": + return None + return int(value) + + +def _rpc_error(request_id: Any, code: int, message: str) -> dict[str, Any]: + return {"jsonrpc": "2.0", "id": request_id, "error": {"code": code, "message": message}} + + +def _read_message(stream: Any) -> dict[str, Any] | None: + line = stream.readline() + if not line: + return None + if line.lower().startswith(b"content-length:"): + length = int(line.split(b":", 1)[1].strip()) + while True: + header = stream.readline() + if header in {b"\r\n", b"\n", b""}: + break + body = stream.read(length) + return json.loads(body.decode("utf-8")) + return json.loads(line.decode("utf-8")) + + +def _write_message(stream: Any, message: dict[str, Any]) -> None: + body = json.dumps(message, separators=(",", ":"), sort_keys=True).encode("utf-8") + stream.write(f"Content-Length: {len(body)}\r\n\r\n".encode("ascii")) + stream.write(body) + stream.flush() + + +def _package_version() -> str: + try: + return version("codebase-graph") + except PackageNotFoundError: + return "0.1.0" diff --git a/src/memory/__init__.py b/src/codebase_graph/memory/__init__.py similarity index 100% rename from src/memory/__init__.py rename to src/codebase_graph/memory/__init__.py diff --git a/src/ontology/__init__.py b/src/codebase_graph/ontology/__init__.py similarity index 100% rename from src/ontology/__init__.py rename to src/codebase_graph/ontology/__init__.py diff --git a/src/ontology/ontology.py b/src/codebase_graph/ontology/ontology.py similarity index 100% rename from src/ontology/ontology.py rename to src/codebase_graph/ontology/ontology.py diff --git a/src/reasoning/__init__.py b/src/codebase_graph/reasoning/__init__.py similarity index 100% rename from src/reasoning/__init__.py rename to src/codebase_graph/reasoning/__init__.py diff --git a/src/reasoning/context_builder.py b/src/codebase_graph/reasoning/context_builder.py similarity index 98% rename from src/reasoning/context_builder.py rename to src/codebase_graph/reasoning/context_builder.py index 1f0b49a..9c1b3fa 100644 --- a/src/reasoning/context_builder.py +++ b/src/codebase_graph/reasoning/context_builder.py @@ -3,8 +3,8 @@ from dataclasses import dataclass, field from typing import Any -from db.schema import quote_identifier -from ontology import CONTEXT_PROFILES, RELATION_TYPES +from codebase_graph.db.schema import quote_identifier +from codebase_graph.ontology import CONTEXT_PROFILES, RELATION_TYPES DEFAULT_CONTEXT_LIMIT = 3 diff --git a/src/retrieval/__init__.py b/src/codebase_graph/retrieval/__init__.py similarity index 100% rename from src/retrieval/__init__.py rename to src/codebase_graph/retrieval/__init__.py diff --git a/src/retrieval/search.py b/src/codebase_graph/retrieval/search.py similarity index 98% rename from src/retrieval/search.py rename to src/codebase_graph/retrieval/search.py index dd24fe2..396c9ee 100644 --- a/src/retrieval/search.py +++ b/src/codebase_graph/retrieval/search.py @@ -3,8 +3,8 @@ from dataclasses import dataclass, field from typing import Any -from ontology import CONTEXT_PROFILES, SEARCH_INDEXES -from reasoning.context_builder import CompactContextBuilder, ContextNode, DEFAULT_CONTEXT_BUDGET, DEFAULT_CONTEXT_LIMIT +from codebase_graph.ontology import CONTEXT_PROFILES, SEARCH_INDEXES +from codebase_graph.reasoning.context_builder import CompactContextBuilder, ContextNode, DEFAULT_CONTEXT_BUDGET, DEFAULT_CONTEXT_LIMIT DEFAULT_SEARCH_LIMIT = 3 diff --git a/src/codebase_graph/setup/__init__.py b/src/codebase_graph/setup/__init__.py new file mode 100644 index 0000000..02b4e36 --- /dev/null +++ b/src/codebase_graph/setup/__init__.py @@ -0,0 +1,24 @@ +"""Production setup orchestration for repository graph bootstrapping.""" + +from .orchestrator import SetupError, SetupOptions, SetupResult, run_setup +from .state import ( + CONFIG_NAME, + DEFAULT_STATE_DIR, + MANIFEST_NAME, + SetupPaths, + derive_setup_paths, + load_setup_config, +) + +__all__ = [ + "CONFIG_NAME", + "DEFAULT_STATE_DIR", + "MANIFEST_NAME", + "SetupError", + "SetupOptions", + "SetupPaths", + "SetupResult", + "derive_setup_paths", + "load_setup_config", + "run_setup", +] diff --git a/src/codebase_graph/setup/instructions.py b/src/codebase_graph/setup/instructions.py new file mode 100644 index 0000000..9f0fb61 --- /dev/null +++ b/src/codebase_graph/setup/instructions.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +START_MARKER = "" +END_MARKER = "" + + +@dataclass(frozen=True, slots=True) +class InstructionResult: + action: str + path: str | None + + def as_dict(self) -> dict[str, str | None]: + return {"action": self.action, "path": self.path} + + +def upsert_instruction_block( + repo_root: Path, + *, + target: str = "auto", + server_name: str, + config_path: Path, +) -> InstructionResult: + if target == "skip": + return InstructionResult("skipped", None) + path = _select_instruction_path(repo_root, target) + existing = path.read_text(encoding="utf-8") if path.exists() else "" + block = _instruction_block(server_name=server_name, config_path=config_path) + next_text, action = _upsert_block(existing, block, created=not path.exists()) + if next_text == existing: + return InstructionResult("unchanged", path.as_posix()) + path.write_text(next_text, encoding="utf-8") + return InstructionResult(action, path.as_posix()) + + +def remove_instruction_block(path: Path) -> bool: + if not path.exists(): + return False + existing = path.read_text(encoding="utf-8") + start = existing.find(START_MARKER) + end = existing.find(END_MARKER) + if start == -1 or end == -1 or end < start: + return False + after_end = end + len(END_MARKER) + next_text = (existing[:start].rstrip() + "\n\n" + existing[after_end:].lstrip()).strip() + "\n" + path.write_text(next_text, encoding="utf-8") + return True + + +def _select_instruction_path(repo_root: Path, target: str) -> Path: + if target == "agents": + return repo_root / "AGENTS.md" + if target == "claude": + return repo_root / "CLAUDE.md" + if target != "auto": + raise ValueError(f"Unsupported instruction target: {target}") + agents = repo_root / "AGENTS.md" + claude = repo_root / "CLAUDE.md" + if agents.exists(): + return agents + if claude.exists(): + return claude + return agents + + +def _instruction_block(*, server_name: str, config_path: Path) -> str: + return ( + f"{START_MARKER}\n" + "## codebaseGraph workflow\n" + f"- Use the `{server_name}` MCP server for repository graph search, schema, and compact context before answering repo-structure questions.\n" + "- Prefer `graph_search` for symbols, paths, docs, and setup instructions; follow with `graph_context` when relationships or nearby evidence matter.\n" + "- Use `graph_schema` or `graph_query_helpers` before writing raw graph queries, and keep `graph_query` read-only.\n" + f"- Refresh the graph with `codebase-graph setup --repo-root .` when files change materially. Setup config: `{config_path.as_posix()}`.\n" + f"{END_MARKER}\n" + ) + + +def _upsert_block(existing: str, block: str, *, created: bool) -> tuple[str, str]: + if not existing.strip(): + return block, "created" + start = existing.find(START_MARKER) + end = existing.find(END_MARKER) + if start != -1 and end != -1 and end > start: + after_end = end + len(END_MARKER) + return _join_sections(existing[:start], block, existing[after_end:]), "updated" + separator = "" if existing.endswith("\n") else "\n" + action = "created" if created else "updated" + return existing.rstrip() + separator + "\n" + block, action + + +def _join_sections(prefix: str, block: str, suffix: str) -> str: + sections = [section.strip() for section in (prefix, block, suffix) if section.strip()] + return "\n\n".join(sections) + "\n" diff --git a/src/codebase_graph/setup/mcp_config.py b/src/codebase_graph/setup/mcp_config.py new file mode 100644 index 0000000..87d4432 --- /dev/null +++ b/src/codebase_graph/setup/mcp_config.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +import json +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from .state import MCP_SERVER_NAME + + +@dataclass(frozen=True, slots=True) +class McpConfigResult: + action: str + client: str + path: str | None + server_name: str + entry: dict[str, Any] + + def as_dict(self) -> dict[str, Any]: + return { + "action": self.action, + "client": self.client, + "path": self.path, + "server_name": self.server_name, + "entry": self.entry, + } + + +def configure_mcp_client( + *, + client: str, + config_path: str | Path | None, + setup_config_path: Path, + dry_run: bool = False, + skip: bool = False, +) -> McpConfigResult: + entry = server_entry(setup_config_path) + if skip or client == "none": + return McpConfigResult("skipped", client, None, MCP_SERVER_NAME, entry) + path = Path(config_path).expanduser().resolve() if config_path is not None else default_config_path(client) + next_payload, action = _next_config_payload(path, entry) + if dry_run: + return McpConfigResult("dry_run", client, path.as_posix(), MCP_SERVER_NAME, entry) + path.parent.mkdir(parents=True, exist_ok=True) + tmp_path = path.with_suffix(path.suffix + ".tmp") + with tmp_path.open("w", encoding="utf-8") as handle: + json.dump(next_payload, handle, indent=2, sort_keys=True) + handle.write("\n") + os.replace(tmp_path, path) + return McpConfigResult(action, client, path.as_posix(), MCP_SERVER_NAME, entry) + + +def server_entry(setup_config_path: Path) -> dict[str, Any]: + return { + "command": "codebase-graph", + "args": ["mcp", "serve", "--config", setup_config_path.as_posix()], + } + + +def default_config_path(client: str) -> Path: + if client == "codex": + base = Path(os.environ.get("CODEX_HOME", Path.home() / ".codex")) + return base / "mcp.json" + if client == "claude": + mac_path = Path.home() / "Library" / "Application Support" / "Claude" / "claude_desktop_config.json" + if mac_path.parent.exists(): + return mac_path + return Path.home() / ".config" / "claude" / "claude_desktop_config.json" + raise ValueError(f"Unsupported MCP client: {client}") + + +def _next_config_payload(path: Path, entry: dict[str, Any]) -> tuple[dict[str, Any], str]: + payload = _read_json(path) + servers = payload.setdefault("mcpServers", {}) + previous = servers.get(MCP_SERVER_NAME) + servers[MCP_SERVER_NAME] = entry + if previous is None: + return payload, "created" + if previous == entry: + return payload, "unchanged" + return payload, "updated" + + +def _read_json(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + with path.open("r", encoding="utf-8") as handle: + payload = json.load(handle) + if not isinstance(payload, dict): + raise ValueError(f"MCP config must contain a JSON object: {path}") + return payload diff --git a/src/codebase_graph/setup/orchestrator.py b/src/codebase_graph/setup/orchestrator.py new file mode 100644 index 0000000..e431c46 --- /dev/null +++ b/src/codebase_graph/setup/orchestrator.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from codebase_graph.ingest import GraphMaterializer + +from .instructions import InstructionResult, upsert_instruction_block +from .mcp_config import McpConfigResult, configure_mcp_client, server_entry +from .preflight import validate_ladybug_runtime +from .state import MCP_SERVER_NAME, SetupPaths, build_setup_config, derive_setup_paths, write_setup_config + + +class SetupError(RuntimeError): + pass + + +@dataclass(frozen=True, slots=True) +class SetupOptions: + repo_root: str | Path = "." + mcp_client: str = "codex" + mcp_config_path: str | Path | None = None + skip_mcp_config: bool = False + dry_run: bool = False + instructions_target: str = "auto" + mode: str = "changed" + + +@dataclass(frozen=True, slots=True) +class SetupResult: + paths: SetupPaths + config_action: str + materialization: Any + mcp_config: McpConfigResult + instructions: InstructionResult + legacy_state_detected: bool + + def as_dict(self) -> dict[str, Any]: + return { + **self.paths.as_dict(), + "config_action": self.config_action, + "legacy_state_detected": self.legacy_state_detected, + "mcp_config": self.mcp_config.as_dict(), + "instructions": self.instructions.as_dict(), + "materialization": _materialization_payload(self.materialization), + } + + +def run_setup(options: SetupOptions) -> SetupResult: + try: + paths = derive_setup_paths(options.repo_root) + validate_ladybug_runtime() + paths.state_dir.mkdir(parents=True, exist_ok=True) + mcp_entry = server_entry(paths.config_path) + config_payload = build_setup_config(paths, mcp_command=[mcp_entry["command"], *mcp_entry["args"]]) + config_action = write_setup_config(paths.config_path, config_payload) + instructions = upsert_instruction_block( + paths.repo_root, + target=options.instructions_target, + server_name=MCP_SERVER_NAME, + config_path=paths.config_path, + ) + materializer = GraphMaterializer( + paths.repo_root, + db_path=paths.db_path, + manifest_path=paths.manifest_path, + include_fts=True, + repository_label=paths.repo_name, + ) + materialization = materializer.materialize(mode=options.mode) # type: ignore[arg-type] + mcp_result = configure_mcp_client( + client=options.mcp_client, + config_path=options.mcp_config_path, + setup_config_path=paths.config_path, + dry_run=options.dry_run, + skip=options.skip_mcp_config, + ) + except Exception as exc: + if isinstance(exc, SetupError): + raise + raise SetupError(str(exc)) from exc + return SetupResult( + paths=paths, + config_action=config_action, + materialization=materialization, + mcp_config=mcp_result, + instructions=instructions, + legacy_state_detected=(paths.repo_root / ".codebase_graph").exists(), + ) + + +def _materialization_payload(result: Any) -> dict[str, Any]: + return { + "mode": getattr(result, "mode"), + "scanned": getattr(result, "scanned"), + "rebuilt": getattr(result, "rebuilt"), + "skipped": getattr(result, "skipped"), + "deleted": getattr(result, "deleted"), + "diagnostics": list(getattr(result, "diagnostics")), + "manifest_path": getattr(result, "manifest_path"), + "rebuilt_paths": list(getattr(result, "rebuilt_paths")), + "skipped_paths": list(getattr(result, "skipped_paths")), + "deleted_paths": list(getattr(result, "deleted_paths")), + "graph_summary": dict(getattr(result, "graph_summary")), + } diff --git a/src/codebase_graph/setup/preflight.py b/src/codebase_graph/setup/preflight.py new file mode 100644 index 0000000..0a3aca5 --- /dev/null +++ b/src/codebase_graph/setup/preflight.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +import tempfile +from pathlib import Path + +from codebase_graph.db import LadybugUnavailableError, create_ladybug_database + + +def validate_ladybug_runtime() -> None: + """Fail before setup creates repo state if LadyBugDB cannot create a graph DB.""" + try: + import real_ladybug # noqa: F401 + except ImportError as exc: + raise LadybugUnavailableError( + "LadyBugDB is required for codebaseGraph setup. Install a package build that includes `real_ladybug`." + ) from exc + + with tempfile.TemporaryDirectory(prefix="codebase-graph-preflight-") as temp_dir: + db_path = Path(temp_dir) / "preflight.ldb" + store = create_ladybug_database(db_path, include_fts=False) + store.close() diff --git a/src/codebase_graph/setup/state.py b/src/codebase_graph/setup/state.py new file mode 100644 index 0000000..2e0b67c --- /dev/null +++ b/src/codebase_graph/setup/state.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +import json +import os +from dataclasses import dataclass +from importlib.metadata import PackageNotFoundError, version +from pathlib import Path +from typing import Any + +from codebase_graph.ontology import ONTOLOGY_VERSION + +DEFAULT_STATE_DIR = ".codebaseGraph" +CONFIG_NAME = "config.json" +MANIFEST_NAME = "manifest.json" +MCP_SERVER_NAME = "codebaseGraph" + + +@dataclass(frozen=True, slots=True) +class SetupPaths: + repo_root: Path + repo_name: str + state_dir: Path + db_path: Path + manifest_path: Path + config_path: Path + + def as_dict(self) -> dict[str, str]: + return { + "repo_root": self.repo_root.as_posix(), + "repo_name": self.repo_name, + "state_dir": self.state_dir.as_posix(), + "db_path": self.db_path.as_posix(), + "manifest_path": self.manifest_path.as_posix(), + "config_path": self.config_path.as_posix(), + } + + +def derive_setup_paths(repo_root: str | Path) -> SetupPaths: + root = Path(repo_root).expanduser().resolve() + if not root.exists(): + raise FileNotFoundError(f"Repository root does not exist: {root}") + if not root.is_dir(): + raise NotADirectoryError(f"Repository root is not a directory: {root}") + repo_name = _repo_name(root) + state_dir = root / DEFAULT_STATE_DIR + return SetupPaths( + repo_root=root, + repo_name=repo_name, + state_dir=state_dir, + db_path=state_dir / f"{repo_name}_graph.ldb", + manifest_path=state_dir / MANIFEST_NAME, + config_path=state_dir / CONFIG_NAME, + ) + + +def build_setup_config(paths: SetupPaths, *, mcp_command: list[str]) -> dict[str, Any]: + return { + "schema_version": 1, + "repo_root": paths.repo_root.as_posix(), + "repo_name": paths.repo_name, + "state_dir": paths.state_dir.as_posix(), + "database_path": paths.db_path.as_posix(), + "manifest_path": paths.manifest_path.as_posix(), + "ontology_version": ONTOLOGY_VERSION, + "package_version": _package_version(), + "mcp": { + "server_name": MCP_SERVER_NAME, + "command": list(mcp_command), + }, + } + + +def load_setup_config(path: str | Path) -> dict[str, Any]: + config_path = Path(path).expanduser().resolve() + with config_path.open("r", encoding="utf-8") as handle: + payload = json.load(handle) + _validate_setup_config(payload, config_path) + return payload + + +def write_setup_config(path: Path, payload: dict[str, Any]) -> str: + previous = _read_json_if_exists(path) + action = "created" + if previous == payload: + return "unchanged" + if previous is not None: + action = "updated" + path.parent.mkdir(parents=True, exist_ok=True) + tmp_path = path.with_suffix(path.suffix + ".tmp") + with tmp_path.open("w", encoding="utf-8") as handle: + json.dump(payload, handle, indent=2, sort_keys=True) + handle.write("\n") + os.replace(tmp_path, path) + return action + + +def _repo_name(root: Path) -> str: + name = root.name.strip() + if name: + return _safe_name(name) + return "repository" + + +def _safe_name(value: str) -> str: + normalized = "".join(character if character.isalnum() or character in {"-", "_"} else "_" for character in value) + return normalized.strip("._-") or "repository" + + +def _package_version() -> str: + try: + return version("codebase-graph") + except PackageNotFoundError: + return "0.1.0" + + +def _read_json_if_exists(path: Path) -> dict[str, Any] | None: + if not path.exists(): + return None + with path.open("r", encoding="utf-8") as handle: + return json.load(handle) + + +def _validate_setup_config(payload: dict[str, Any], path: Path) -> None: + required = ("repo_root", "repo_name", "database_path", "manifest_path") + missing = [key for key in required if not payload.get(key)] + if missing: + joined = ", ".join(missing) + raise ValueError(f"Invalid codebaseGraph setup config at {path}: missing {joined}") diff --git a/src/tests/__init__.py b/src/tests/__init__.py deleted file mode 100644 index 6f9bec8..0000000 --- a/src/tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Tests for the fresh src module layout.""" diff --git a/src/tests/test_graph_builder.py b/tests/test_graph_builder.py similarity index 98% rename from src/tests/test_graph_builder.py rename to tests/test_graph_builder.py index 6ac7a88..01c66bb 100644 --- a/src/tests/test_graph_builder.py +++ b/tests/test_graph_builder.py @@ -1,7 +1,7 @@ from __future__ import annotations -from extract import CaptureRecord, GraphBuilder, ParseBundle -from ontology import PARSER_NODE_MAPPINGS +from codebase_graph.extract import CaptureRecord, GraphBuilder, ParseBundle +from codebase_graph.ontology import PARSER_NODE_MAPPINGS def test_graph_builder_maps_python_ast_shaped_tree_to_ontology() -> None: diff --git a/src/tests/test_materializer.py b/tests/test_materializer.py similarity index 95% rename from src/tests/test_materializer.py rename to tests/test_materializer.py index c4b7e0a..bf8c20c 100644 --- a/src/tests/test_materializer.py +++ b/tests/test_materializer.py @@ -5,16 +5,16 @@ import pytest -import ingest.materializer as materializer_module -from db import LadybugCodeGraphStore -from ingest import ( +import codebase_graph.ingest.materializer as materializer_module +from codebase_graph.db import LadybugCodeGraphStore +from codebase_graph.ingest import ( GraphMaterializer, ManifestEntry, MaterializationManifest, SourceSnapshot, TreeSitterPythonParser, ) -from ontology import ONTOLOGY_NAME +from codebase_graph.ontology import ONTOLOGY_NAME def test_manifest_diff_tracks_added_modified_unchanged_and_deleted(tmp_path: Path) -> None: @@ -85,7 +85,7 @@ def fake_walk(root: Path) -> object: for dirname in dirnames: yield (Path(root) / dirname).as_posix(), [], ["app.py"] - monkeypatch.setattr("ingest.materializer.os.walk", fake_walk) + monkeypatch.setattr("codebase_graph.ingest.materializer.os.walk", fake_walk) materializer = GraphMaterializer(tmp_path, db_path=":memory:", manifest_path=tmp_path / "manifest.json", store=object()) snapshots, diagnostics = materializer._scan_source_files() @@ -112,14 +112,17 @@ def test_full_materialization_writes_python_graph_to_ladybug(tmp_path: Path) -> ) result = materializer.materialize(mode="full") - assert result.rebuilt == 3 + assert result.rebuilt == 4 assert result.deleted == 0 - assert result.graph_summary["partition_count"] == 3 + assert result.graph_summary["partition_count"] == 4 assert _labels(materializer, "File") == { "__init__.py", + "README.md", "cli.py", "service.py", } + assert "README.md" in _labels(materializer, "DocumentationSource") + assert _labels(materializer, "DocumentationChunk") assert "SampleService" in _labels(materializer, "Class") assert "run" in _labels(materializer, "Method") assert {"helper", "main"} <= _labels(materializer, "Function") @@ -173,7 +176,7 @@ def test_changed_materialization_only_rebuilds_changed_files(tmp_path: Path) -> (source_root / "sample_project" / "cli.py").unlink() fourth = materializer.materialize(mode="changed") - assert first.rebuilt == 3 + assert first.rebuilt == 4 assert second.rebuilt == 0 assert third.rebuilt == 1 assert third.rebuilt_paths == ("sample_project/service.py",) @@ -276,7 +279,7 @@ def fail_clear_graph(self: LadybugCodeGraphStore) -> None: result = GraphMaterializer(source_root, db_path=db_path, manifest_path=manifest_path, include_fts=False).materialize(mode="changed") assert result.mode == "changed" - assert result.rebuilt == 3 + assert result.rebuilt == 4 assert not marker_path.exists() reader = GraphMaterializer(source_root, db_path=db_path, manifest_path=manifest_path, include_fts=False) assert "SampleService" in _labels(reader, "Class") diff --git a/src/tests/test_ontology.py b/tests/test_ontology.py similarity index 99% rename from src/tests/test_ontology.py rename to tests/test_ontology.py index 63f0b48..06c4fd0 100644 --- a/src/tests/test_ontology.py +++ b/tests/test_ontology.py @@ -3,7 +3,7 @@ import json import re -from ontology import ( +from codebase_graph.ontology import ( ONTOLOGY_NAME, PARSER_NODE_MAPPINGS, QUERY_HELPERS, diff --git a/src/tests/test_schema.py b/tests/test_schema.py similarity index 97% rename from src/tests/test_schema.py rename to tests/test_schema.py index 6f7cf92..a6593a5 100644 --- a/src/tests/test_schema.py +++ b/tests/test_schema.py @@ -2,8 +2,8 @@ import pytest -from core import CodeGraph, GraphEdge, GraphNode -from db import ( +from codebase_graph.core import CodeGraph, GraphEdge, GraphNode +from codebase_graph.db import ( LadybugCodeGraphStore, build_ladybug_schema, build_ladybug_schema_statements, @@ -11,7 +11,7 @@ ladybug_type, quote_identifier, ) -from ontology import NODE_TYPES, RELATION_TYPES, SEARCH_INDEXES +from codebase_graph.ontology import NODE_TYPES, RELATION_TYPES, SEARCH_INDEXES def test_ladybug_schema_declares_all_ontology_nodes_and_edge_nodes() -> None: diff --git a/src/tests/test_search.py b/tests/test_search.py similarity index 97% rename from src/tests/test_search.py rename to tests/test_search.py index 2db267b..93415bf 100644 --- a/src/tests/test_search.py +++ b/tests/test_search.py @@ -7,10 +7,10 @@ import pytest -from cli import main as cli_main -from ingest import GraphMaterializer -from reasoning import CompactContextBuilder -from retrieval.search import SearchHit, SearchRequest, SearchService +from codebase_graph.cli import main as cli_main +from codebase_graph.ingest import GraphMaterializer +from codebase_graph.reasoning import CompactContextBuilder +from codebase_graph.retrieval.search import SearchHit, SearchRequest, SearchService class _Result: diff --git a/tests/test_setup_workflow.py b/tests/test_setup_workflow.py new file mode 100644 index 0000000..278e8d0 --- /dev/null +++ b/tests/test_setup_workflow.py @@ -0,0 +1,192 @@ +from __future__ import annotations + +import json +from pathlib import Path + +try: + import tomllib +except ImportError: # pragma: no cover - Python 3.10 compatibility + import tomli as tomllib + +import pytest + +from codebase_graph.cli import main as cli_main +from codebase_graph.db import LadybugUnavailableError +from codebase_graph.mcp.server import McpGraphServer, handle_tool_call +from codebase_graph.setup import SetupError, SetupOptions, run_setup +from codebase_graph.setup.instructions import END_MARKER, START_MARKER +from codebase_graph.setup.mcp_config import configure_mcp_client + + +def test_setup_cli_creates_state_db_mcp_config_instructions_and_searchable_docs( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + pytest.importorskip("tree_sitter") + pytest.importorskip("tree_sitter_python") + pytest.importorskip("real_ladybug") + repo_root = _fresh_repo(tmp_path) + mcp_config_path = tmp_path / "mcp.json" + + exit_code = cli_main( + [ + "setup", + "--repo-root", + repo_root.as_posix(), + "--mcp-client", + "codex", + "--mcp-config-path", + mcp_config_path.as_posix(), + ] + ) + first_output = json.loads(capsys.readouterr().out) + + assert exit_code == 0 + assert first_output["state_dir"] == (repo_root / ".codebaseGraph").as_posix() + assert first_output["db_path"] == (repo_root / ".codebaseGraph" / "fresh_repo_graph.ldb").as_posix() + assert Path(first_output["db_path"]).exists() + assert Path(first_output["config_path"]).exists() + assert first_output["materialization"]["rebuilt"] == 4 + assert first_output["instructions"]["path"] == (repo_root / "AGENTS.md").as_posix() + assert first_output["mcp_config"]["action"] == "created" + + agents_text = (repo_root / "AGENTS.md").read_text(encoding="utf-8") + assert agents_text.count(START_MARKER) == 1 + assert agents_text.count(END_MARKER) == 1 + mcp_payload = json.loads(mcp_config_path.read_text(encoding="utf-8")) + assert "otherServer" not in mcp_payload.get("mcpServers", {}) + assert mcp_payload["mcpServers"]["codebaseGraph"]["args"] == [ + "mcp", + "serve", + "--config", + (repo_root / ".codebaseGraph" / "config.json").as_posix(), + ] + + second_exit_code = cli_main( + [ + "setup", + "--repo-root", + repo_root.as_posix(), + "--mcp-config-path", + mcp_config_path.as_posix(), + ] + ) + second_output = json.loads(capsys.readouterr().out) + + assert second_exit_code == 0 + assert second_output["config_action"] == "unchanged" + assert second_output["instructions"]["action"] == "unchanged" + assert second_output["mcp_config"]["action"] == "unchanged" + assert (repo_root / "AGENTS.md").read_text(encoding="utf-8").count(START_MARKER) == 1 + + server = McpGraphServer.from_paths(config_path=repo_root / ".codebaseGraph" / "config.json") + docs_payload = handle_tool_call( + "graph_search", + {"query": "codebaseGraph workflow", "profile": "docs", "limit": 5}, + runtime=server.runtime, + ) + symbol_payload = handle_tool_call( + "graph_search", + {"query": "SampleService", "profile": "brief", "limit": 3}, + runtime=server.runtime, + ) + + assert any(hit["path"] == "AGENTS.md" for hit in docs_payload["results"]) + assert any(hit["label"] == "SampleService" for hit in symbol_payload["results"]) + + +def test_mcp_config_dry_run_preserves_existing_servers(tmp_path: Path) -> None: + config_path = tmp_path / "mcp.json" + config_path.write_text( + json.dumps({"mcpServers": {"otherServer": {"command": "other", "args": []}}}), + encoding="utf-8", + ) + setup_config_path = tmp_path / ".codebaseGraph" / "config.json" + + dry_run = configure_mcp_client( + client="codex", + config_path=config_path, + setup_config_path=setup_config_path, + dry_run=True, + ) + + assert dry_run.action == "dry_run" + assert "codebaseGraph" not in json.loads(config_path.read_text(encoding="utf-8"))["mcpServers"] + + written = configure_mcp_client( + client="codex", + config_path=config_path, + setup_config_path=setup_config_path, + ) + payload = json.loads(config_path.read_text(encoding="utf-8")) + + assert written.action == "created" + assert set(payload["mcpServers"]) == {"otherServer", "codebaseGraph"} + + +def test_setup_preflight_failure_stops_before_state_creation(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + repo_root = _fresh_repo(tmp_path) + + def fail_preflight() -> None: + raise LadybugUnavailableError("missing LadyBugDB") + + monkeypatch.setattr("codebase_graph.setup.orchestrator.validate_ladybug_runtime", fail_preflight) + + with pytest.raises(SetupError, match="missing LadyBugDB"): + run_setup(SetupOptions(repo_root=repo_root, mcp_client="none")) + + assert not (repo_root / ".codebaseGraph").exists() + + +def test_mcp_graph_query_rejects_write_like_statements(tmp_path: Path) -> None: + pytest.importorskip("tree_sitter") + pytest.importorskip("tree_sitter_python") + pytest.importorskip("real_ladybug") + repo_root = _fresh_repo(tmp_path) + result = run_setup(SetupOptions(repo_root=repo_root, mcp_client="none", instructions_target="skip")) + server = McpGraphServer.from_paths(config_path=result.paths.config_path) + + with pytest.raises(ValueError, match="read-only"): + handle_tool_call( + "graph_query", + {"statement": "MATCH (n) DELETE n"}, + runtime=server.runtime, + ) + + +def test_setup_invalid_repo_root_exits_nonzero(tmp_path: Path) -> None: + missing = tmp_path / "missing" + + with pytest.raises(SystemExit) as exc_info: + cli_main(["setup", "--repo-root", missing.as_posix(), "--mcp-client", "none"]) + + assert exc_info.value.code == 2 + + +def test_packaging_requires_ladybug_and_namespaced_package_discovery() -> None: + payload = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8")) + + assert "real_ladybug" in payload["project"]["dependencies"] + assert payload["project"]["scripts"]["codebase-graph"] == "codebase_graph.cli:main" + assert payload["project"]["scripts"]["codebase-graph-mcp"] == "codebase_graph.mcp.server:main" + assert payload["tool"]["setuptools"]["packages"]["find"]["include"] == ["codebase_graph*"] + + +def _fresh_repo(tmp_path: Path) -> Path: + repo_root = tmp_path / "fresh_repo" + package = repo_root / "sample_project" + package.mkdir(parents=True) + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "service.py").write_text( + "class SampleService:\n" + " def run(self) -> str:\n" + " return helper()\n\n" + "def helper() -> str:\n" + " return 'ok'\n", + encoding="utf-8", + ) + (repo_root / "README.md").write_text( + "# Fresh Repo\n\nThis repository documents the SampleService workflow.\n", + encoding="utf-8", + ) + return repo_root From 2b4aefbeaf7cfd59da7500906ea5f564bfa53492 Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Mon, 25 May 2026 11:31:32 +0930 Subject: [PATCH 11/53] refactor: improve library modularity --- src/codebase_graph/cli/__init__.py | 8 +- src/codebase_graph/db/__init__.py | 6 + src/codebase_graph/db/query.py | 186 ++++++++++++++++++ src/codebase_graph/extract/__init__.py | 20 +- src/codebase_graph/extract/graph_builder.py | 142 ++++++++----- src/codebase_graph/ingest/__init__.py | 14 +- src/codebase_graph/ingest/materializer.py | 44 +++-- .../ingest/tree_sitter_parser.py | 93 ++++++++- src/codebase_graph/ontology/ontology.py | 31 ++- src/codebase_graph/paths.py | 55 ++++++ .../reasoning/context_builder.py | 58 ++---- src/codebase_graph/retrieval/search.py | 50 ++--- src/codebase_graph/setup/__init__.py | 4 + src/codebase_graph/setup/state.py | 66 ++----- tests/test_graph_builder.py | 21 +- tests/test_materializer.py | 32 +++ tests/test_ontology.py | 15 ++ tests/test_search.py | 82 ++++++++ 18 files changed, 707 insertions(+), 220 deletions(-) create mode 100644 src/codebase_graph/db/query.py create mode 100644 src/codebase_graph/paths.py diff --git a/src/codebase_graph/cli/__init__.py b/src/codebase_graph/cli/__init__.py index d0b88e3..85f50ad 100644 --- a/src/codebase_graph/cli/__init__.py +++ b/src/codebase_graph/cli/__init__.py @@ -17,8 +17,8 @@ def main(argv: Sequence[str] | None = None) -> int: materialize_parser = subparsers.add_parser("materialize", help="Materialize the code graph") materialize_parser.add_argument("--source-root", default=".", help="Repository or source root to scan") - materialize_parser.add_argument("--db", default=None, help="LadybugDB path; defaults under .codebase_graph") - materialize_parser.add_argument("--manifest", default=None, help="Manifest path; defaults under .codebase_graph") + materialize_parser.add_argument("--db", default=None, help="LadybugDB path; defaults under .codebaseGraph") + materialize_parser.add_argument("--manifest", default=None, help="Manifest path; defaults under .codebaseGraph") materialize_parser.add_argument("--mode", choices=("full", "changed"), default="changed") materialize_parser.add_argument("--no-fts", action="store_true", help="Skip FTS index creation") @@ -114,8 +114,8 @@ def main(argv: Sequence[str] | None = None) -> int: def _add_search_arguments(parser: argparse.ArgumentParser) -> None: parser.add_argument("query", help="Search query") parser.add_argument("--source-root", default=".", help="Repository or source root to search") - parser.add_argument("--db", default=None, help="LadybugDB path; defaults under .codebase_graph") - parser.add_argument("--manifest", default=None, help="Manifest path; defaults under .codebase_graph") + parser.add_argument("--db", default=None, help="LadybugDB path; defaults under .codebaseGraph") + parser.add_argument("--manifest", default=None, help="Manifest path; defaults under .codebaseGraph") parser.add_argument("--limit", type=int, default=3, help="Maximum search hits to return") parser.add_argument("--profile", choices=sorted(CONTEXT_PROFILES), default="brief", help="Context profile") parser.add_argument("--budget", type=int, default=600, help="Approximate per-hit context character budget") diff --git a/src/codebase_graph/db/__init__.py b/src/codebase_graph/db/__init__.py index 6213bea..c3beabe 100644 --- a/src/codebase_graph/db/__init__.py +++ b/src/codebase_graph/db/__init__.py @@ -1,14 +1,20 @@ """Storage adapters, schema management, and migrations.""" +from .query import GraphNeighbor, GraphQueryAdapter, LadybugGraphQueryAdapter, SearchIndexRow, graph_query_adapter from .schema import build_ladybug_schema, build_ladybug_schema_statements, ladybug_type, quote_identifier from .store import LadybugCodeGraphStore, LadybugUnavailableError, create_ladybug_database __all__ = [ + "GraphNeighbor", + "GraphQueryAdapter", "LadybugCodeGraphStore", + "LadybugGraphQueryAdapter", "LadybugUnavailableError", + "SearchIndexRow", "build_ladybug_schema", "build_ladybug_schema_statements", "create_ladybug_database", + "graph_query_adapter", "ladybug_type", "quote_identifier", ] diff --git a/src/codebase_graph/db/query.py b/src/codebase_graph/db/query.py new file mode 100644 index 0000000..02a1d98 --- /dev/null +++ b/src/codebase_graph/db/query.py @@ -0,0 +1,186 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Protocol + +from codebase_graph.ontology import get_relation_type + +from .schema import quote_identifier + + +@dataclass(frozen=True, slots=True) +class GraphNeighbor: + node_id: str + node_type: str + label: str + qualified_name: str = "" + path: str = "" + line_start: int | None = None + line_end: int | None = None + summary: str = "" + + +@dataclass(frozen=True, slots=True) +class SearchIndexRow: + id: str + node_type: str + label: str + qualified_name: str = "" + path: str = "" + line_start: int | None = None + line_end: int | None = None + summary: str = "" + score: float = 0.0 + metadata: dict[str, Any] = field(default_factory=dict) + + +class GraphQueryAdapter(Protocol): + def search_index(self, *, node_type: str, index_name: str, query: str, limit: int) -> list[SearchIndexRow]: + ... + + def neighbors( + self, + *, + node_id: str, + node_type: str, + relation: str, + direction: str, + limit: int, + ) -> list[GraphNeighbor]: + ... + + +class LadybugGraphQueryAdapter: + def __init__(self, store: Any) -> None: + self.store = store + + def search_index(self, *, node_type: str, index_name: str, query: str, limit: int) -> list[SearchIndexRow]: + rows = self.store.execute( + _fts_query_statement(node_type=node_type, index_name=index_name), + {"query": query, "top": limit}, + ).get_all() + return [ + SearchIndexRow( + id=_text(_value(row, 0)), + node_type=node_type, + label=_text(_value(row, 1)), + qualified_name=_text(_value(row, 2)), + path=_text(_value(row, 3)), + line_start=_optional_int(_value(row, 4)), + line_end=_optional_int(_value(row, 5)), + summary=_text(_value(row, 6)), + score=float(_value(row, 7) or 0.0), + ) + for row in rows + ] + + def neighbors( + self, + *, + node_id: str, + node_type: str, + relation: str, + direction: str, + limit: int, + ) -> list[GraphNeighbor]: + if direction not in {"outgoing", "incoming"}: + raise ValueError(f"Unsupported relation direction: {direction}") + try: + relation_type = get_relation_type(relation) + except KeyError: + return [] + + if direction == "outgoing": + if node_type not in relation_type.source_types: + return [] + neighbor_types = relation_type.target_types + else: + if node_type not in relation_type.target_types: + return [] + neighbor_types = relation_type.source_types + + neighbors: list[GraphNeighbor] = [] + for neighbor_type in neighbor_types: + remaining = limit - len(neighbors) + if remaining <= 0: + break + rows = self.store.execute( + _neighbor_statement( + node_type=node_type, + neighbor_type=neighbor_type, + relation=relation, + direction=direction, + limit=remaining, + ), + {"node_id": node_id}, + ).get_all() + neighbors.extend(_neighbor_from_row(row, neighbor_type) for row in rows) + return neighbors + + +def graph_query_adapter(store: Any) -> GraphQueryAdapter: + adapter = getattr(store, "graph_query_adapter", None) + if adapter is not None: + return adapter + return LadybugGraphQueryAdapter(store) + + +def _fts_query_statement(*, node_type: str, index_name: str) -> str: + return ( + f"CALL QUERY_FTS_INDEX('{node_type}', '{index_name}', $query, TOP := $top) " + "RETURN node.id, node.label, node.qualified_name, node.path, " + "node.line_start, node.line_end, node.summary, score" + ) + + +def _neighbor_statement( + *, + node_type: str, + neighbor_type: str, + relation: str, + direction: str, + limit: int, +) -> str: + if direction == "outgoing": + return ( + f"MATCH (source:{quote_identifier(node_type)} {{id: $node_id}})" + f"-[:{quote_identifier(f'FROM_{relation}')}]->(edge:{quote_identifier(relation)})" + f"-[:{quote_identifier(f'TO_{relation}')}]->(neighbor:{quote_identifier(neighbor_type)}) " + "RETURN neighbor.id, neighbor.label, neighbor.qualified_name, neighbor.path, " + f"neighbor.line_start, neighbor.line_end, neighbor.summary LIMIT {int(limit)}" + ) + return ( + f"MATCH (neighbor:{quote_identifier(neighbor_type)})" + f"-[:{quote_identifier(f'FROM_{relation}')}]->(edge:{quote_identifier(relation)})" + f"-[:{quote_identifier(f'TO_{relation}')}]->(target:{quote_identifier(node_type)} {{id: $node_id}}) " + "RETURN neighbor.id, neighbor.label, neighbor.qualified_name, neighbor.path, " + f"neighbor.line_start, neighbor.line_end, neighbor.summary LIMIT {int(limit)}" + ) + + +def _neighbor_from_row(row: Any, node_type: str) -> GraphNeighbor: + return GraphNeighbor( + node_id=_text(_value(row, 0)), + node_type=node_type, + label=_text(_value(row, 1)), + qualified_name=_text(_value(row, 2)), + path=_text(_value(row, 3)), + line_start=_optional_int(_value(row, 4)), + line_end=_optional_int(_value(row, 5)), + summary=_text(_value(row, 6)), + ) + + +def _optional_int(value: Any) -> int | None: + return None if value is None else int(value) + + +def _text(value: Any) -> str: + return "" if value is None else str(value) + + +def _value(row: Any, index: int) -> Any: + try: + return row[index] + except IndexError: + return None diff --git a/src/codebase_graph/extract/__init__.py b/src/codebase_graph/extract/__init__.py index 43be001..f54048f 100644 --- a/src/codebase_graph/extract/__init__.py +++ b/src/codebase_graph/extract/__init__.py @@ -1,5 +1,21 @@ """Code entity and relation extraction.""" -from .graph_builder import CaptureRecord, GraphBuilder, GraphBuildResult, ParseBundle +from .graph_builder import ( + CaptureRecord, + CaptureTableRegistry, + CaptureTableResolver, + GraphBuilder, + GraphBuildResult, + ParseBundle, + default_capture_table_registry, +) -__all__ = ["CaptureRecord", "GraphBuilder", "GraphBuildResult", "ParseBundle"] +__all__ = [ + "CaptureRecord", + "CaptureTableRegistry", + "CaptureTableResolver", + "GraphBuilder", + "GraphBuildResult", + "ParseBundle", + "default_capture_table_registry", +] diff --git a/src/codebase_graph/extract/graph_builder.py b/src/codebase_graph/extract/graph_builder.py index 0258e95..9552624 100644 --- a/src/codebase_graph/extract/graph_builder.py +++ b/src/codebase_graph/extract/graph_builder.py @@ -1,7 +1,7 @@ from __future__ import annotations import hashlib -from collections.abc import Iterable, Mapping, Sequence +from collections.abc import Callable, Iterable, Mapping, Sequence from dataclasses import dataclass from pathlib import Path from typing import Any @@ -77,6 +77,75 @@ class ScopeFrame: qualified_name: str +CaptureTableResolver = Callable[[str, ScopeFrame], str | None] + + +class CaptureTableRegistry: + def __init__(self) -> None: + self._exact: dict[str, str | CaptureTableResolver] = {} + self._prefix: list[tuple[str, str | CaptureTableResolver]] = [] + + def register_exact(self, capture_name: str, table: str | CaptureTableResolver) -> None: + self._exact[_normalize_capture_name(capture_name)] = table + + def register_prefix(self, prefix: str, table: str | CaptureTableResolver) -> None: + self._prefix.append((_normalize_capture_name(prefix), table)) + + def table_for(self, capture_name: str, owner: ScopeFrame) -> str | None: + capture = _normalize_capture_name(capture_name) + if not capture: + return None + if capture in self._exact: + return _resolve_capture_table(self._exact[capture], capture, owner) + for prefix, table in self._prefix: + if capture.startswith(prefix): + return _resolve_capture_table(table, capture, owner) + return None + + +def default_capture_table_registry() -> CaptureTableRegistry: + registry = CaptureTableRegistry() + for capture in ("definition.class", "definition.struct", "definition.interface"): + registry.register_exact(capture, "Class") + registry.register_exact("definition.component", "Component") + registry.register_exact("component", "Component") + registry.register_exact("definition.method", "Method") + registry.register_exact("definition.function", _function_capture_table) + registry.register_exact("definition.parameter", "Parameter") + registry.register_exact("parameter", "Parameter") + registry.register_exact("type.return", "ReturnType") + registry.register_exact("return_type", "ReturnType") + for capture in ("type", "type.annotation", "reference.type"): + registry.register_exact(capture, "TypeAnnotation") + registry.register_exact("definition.type_alias", "TypeAlias") + registry.register_exact("definition.constant", "Constant") + registry.register_exact("definition.variable", "Variable") + registry.register_exact("decorator", "Decorator") + registry.register_exact("definition.decorator", "Decorator") + for capture in ("reference.import", "reference.include", "reference.require", "reference.use", "import"): + registry.register_exact(capture, "ImportDeclaration") + registry.register_exact("export", "ExportDeclaration") + registry.register_exact("definition.export", "ExportDeclaration") + registry.register_exact("reference.call", "CallExpression") + registry.register_exact("call", "CallExpression") + registry.register_prefix("query.", "Query") + registry.register_prefix("secret.", "SecretRef") + registry.register_exact("entrypoint.api", "APIEndpoint") + registry.register_exact("endpoint", "APIEndpoint") + registry.register_exact("route", "Route") + registry.register_exact("doc.source", "DocumentationSource") + registry.register_prefix("doc", "DocumentationChunk") + registry.register_exact("literal", "Literal") + registry.register_exact("string", "Literal") + registry.register_exact("number", "Literal") + registry.register_exact("control_flow", "ControlFlowBlock") + registry.register_exact("exception", "ExceptionFlow") + registry.register_exact("raises", "ExceptionFlow") + registry.register_exact("handles", "ExceptionFlow") + registry.register_prefix("reference", "Reference") + return registry + + class GraphBuilder: """Build an ontology graph from tree-sitter-shaped parser output. @@ -92,11 +161,13 @@ def __init__( repository_label: str = "repository", source_root: str | Path = ".", include_syntax_captures: bool = True, + capture_table_registry: CaptureTableRegistry | None = None, ) -> None: self.default_language = default_language self.repository_label = repository_label self.source_root = Path(source_root).as_posix() self.include_syntax_captures = include_syntax_captures + self.capture_table_registry = capture_table_registry or default_capture_table_registry() self._node_types = set(node_type_names()) self._relation_types = set(relation_type_names()) self._graph = CodeGraph() @@ -218,7 +289,7 @@ def _traverse(self, raw_node: Any, owner: ScopeFrame) -> None: node = self._normalize(raw_node) syntax_id = self._syntax_capture(node) next_owner = owner - capture_table = _table_from_capture(node.capture_name, owner) + capture_table = self.capture_table_registry.table_for(node.capture_name, owner) if capture_table is not None: semantic = self._emit_captured_semantic(capture_table, node, owner, syntax_id) @@ -946,58 +1017,21 @@ def _capture_node_type(capture: Mapping[str, Any] | tuple[Any, str]) -> str: def _table_from_capture(capture_name: str, owner: ScopeFrame) -> str | None: - capture = capture_name.lstrip("@") - if not capture: - return None - if capture in {"definition.class", "definition.struct", "definition.interface"}: - return "Class" - if capture == "definition.component" or capture == "component": - return "Component" - if capture == "definition.method": - return "Method" - if capture == "definition.function": - return "Method" if owner.table in {"Class", "Component"} else "Function" - if capture == "definition.parameter" or capture == "parameter": - return "Parameter" - if capture in {"type.return", "return_type"}: - return "ReturnType" - if capture in {"type", "type.annotation", "reference.type"}: - return "TypeAnnotation" - if capture == "definition.type_alias": - return "TypeAlias" - if capture == "definition.constant": - return "Constant" - if capture == "definition.variable": - return "Variable" - if capture in {"decorator", "definition.decorator"}: - return "Decorator" - if capture in {"reference.import", "reference.include", "reference.require", "reference.use", "import"}: - return "ImportDeclaration" - if capture in {"export", "definition.export"}: - return "ExportDeclaration" - if capture in {"reference.call", "call"}: - return "CallExpression" - if capture.startswith("query."): - return "Query" - if capture.startswith("secret."): - return "SecretRef" - if capture in {"entrypoint.api", "endpoint"}: - return "APIEndpoint" - if capture == "route": - return "Route" - if capture == "doc.source": - return "DocumentationSource" - if capture.startswith("doc"): - return "DocumentationChunk" - if capture in {"literal", "string", "number"}: - return "Literal" - if capture == "control_flow": - return "ControlFlowBlock" - if capture in {"exception", "raises", "handles"}: - return "ExceptionFlow" - if capture.startswith("reference"): - return "Reference" - return None + return default_capture_table_registry().table_for(capture_name, owner) + + +def _normalize_capture_name(capture_name: str) -> str: + return capture_name.lstrip("@") + + +def _resolve_capture_table(table: str | CaptureTableResolver, capture: str, owner: ScopeFrame) -> str | None: + if callable(table): + return table(capture, owner) + return table + + +def _function_capture_table(_capture: str, owner: ScopeFrame) -> str: + return "Method" if owner.table in {"Class", "Component"} else "Function" def _import_source_id(owner: ScopeFrame) -> str: diff --git a/src/codebase_graph/ingest/__init__.py b/src/codebase_graph/ingest/__init__.py index 9352568..3aef5f0 100644 --- a/src/codebase_graph/ingest/__init__.py +++ b/src/codebase_graph/ingest/__init__.py @@ -9,7 +9,15 @@ MaterializeMode, SourceSnapshot, ) -from .tree_sitter_parser import ParserUnavailableError, TreeSitterPythonParser, parser_for_language +from .tree_sitter_parser import ( + ParserRegistry, + ParserRegistration, + ParserUnavailableError, + SourceParser, + TreeSitterPythonParser, + default_parser_registry, + parser_for_language, +) from .document_parser import MarkdownDocumentParser __all__ = [ @@ -20,8 +28,12 @@ "MaterializationManifest", "MaterializationResult", "MaterializeMode", + "ParserRegistry", + "ParserRegistration", "ParserUnavailableError", + "SourceParser", "SourceSnapshot", "TreeSitterPythonParser", + "default_parser_registry", "parser_for_language", ] diff --git a/src/codebase_graph/ingest/materializer.py b/src/codebase_graph/ingest/materializer.py index 4af3973..6411a1a 100644 --- a/src/codebase_graph/ingest/materializer.py +++ b/src/codebase_graph/ingest/materializer.py @@ -14,17 +14,14 @@ from codebase_graph.db import LadybugCodeGraphStore, create_ladybug_database from codebase_graph.extract import GraphBuilder from codebase_graph.ontology import ONTOLOGY_NAME +from codebase_graph.paths import DEFAULT_STATE_DIR, derive_graph_state_paths -from .tree_sitter_parser import ParserUnavailableError, parser_for_language +from .tree_sitter_parser import ParserRegistry, ParserUnavailableError, default_parser_registry MaterializeMode = Literal["full", "changed"] MANIFEST_SCHEMA_VERSION = 1 -DEFAULT_STATE_DIR = ".codebase_graph" -DEFAULT_MANIFEST_NAME = "manifest.json" -DEFAULT_DB_NAME = "graph.lbug" PARSER_VERSION = "tree-sitter-python-v1+markdown-docs-v1" -SUPPORTED_SUFFIXES = {".py": "python", ".md": "markdown", ".mdx": "markdown"} EXCLUDED_PARTS = { ".git", ".venv", @@ -34,7 +31,7 @@ "build", "dist", ".codebase_graph", - ".codebaseGraph", + DEFAULT_STATE_DIR, } @@ -94,8 +91,8 @@ class MaterializationManifest: files: Mapping[str, ManifestEntry] = field(default_factory=dict) @classmethod - def empty(cls) -> MaterializationManifest: - return cls(files={}) + def empty(cls, *, parser_version: str = PARSER_VERSION) -> MaterializationManifest: + return cls(parser_version=parser_version, files={}) @classmethod def load(cls, path: Path) -> MaterializationManifest: @@ -129,15 +126,15 @@ def write(self, path: Path) -> None: handle.write("\n") os.replace(tmp_path, path) - def is_compatible(self) -> bool: + def is_compatible(self, *, parser_version: str = PARSER_VERSION) -> bool: return ( self.schema_version == MANIFEST_SCHEMA_VERSION and self.ontology == ONTOLOGY_NAME - and self.parser_version == PARSER_VERSION + and self.parser_version == parser_version ) - def diff(self, current_files: Mapping[str, SourceSnapshot]) -> ManifestDiff: - if not self.is_compatible(): + def diff(self, current_files: Mapping[str, SourceSnapshot], *, parser_version: str = PARSER_VERSION) -> ManifestDiff: + if not self.is_compatible(parser_version=parser_version): return ManifestDiff( added=tuple(sorted(current_files)), modified=(), @@ -206,16 +203,21 @@ def __init__( include_fts: bool = True, repository_label: str | None = None, store: LadybugCodeGraphStore | None = None, + parser_registry: ParserRegistry | None = None, + graph_builder: GraphBuilder | None = None, ) -> None: self.source_root = Path(source_root).resolve() - self.state_dir = self.source_root / DEFAULT_STATE_DIR - self.db_path = _normalize_db_path(db_path if db_path is not None else self.state_dir / DEFAULT_DB_NAME) - self.manifest_path = Path(manifest_path) if manifest_path is not None else self.state_dir / DEFAULT_MANIFEST_NAME + paths = derive_graph_state_paths(self.source_root) + self.state_dir = paths.state_dir + self.db_path = _normalize_db_path(db_path if db_path is not None else paths.db_path) + self.manifest_path = Path(manifest_path) if manifest_path is not None else paths.manifest_path self.include_fts = include_fts self.repository_label = repository_label or self.source_root.name or "repository" self._store = store self._store_injected = store is not None - self.builder = GraphBuilder(repository_label=self.repository_label, source_root=self.source_root) + self.parser_registry = parser_registry or default_parser_registry() + self.parser_version = self.parser_registry.parser_version + self.builder = graph_builder or GraphBuilder(repository_label=self.repository_label, source_root=self.source_root) @property def store(self) -> LadybugCodeGraphStore: @@ -257,7 +259,7 @@ def materialize(self, mode: MaterializeMode = "changed") -> MaterializationResul retained_node_ids: set[str] = set() retained_edge_ids: set[str] = set() else: - diff = previous_manifest.diff(supported) + diff = previous_manifest.diff(supported, parser_version=self.parser_version) if diff.force_rebuild: if self._can_atomic_rebuild(): return self._materialize_full_atomic( @@ -312,7 +314,7 @@ def materialize(self, mode: MaterializeMode = "changed") -> MaterializationResul if path not in set(diff.deleted) | set(diff.rebuild_paths) } next_files.update(rebuilt_entries) - next_manifest = MaterializationManifest(files=next_files) + next_manifest = MaterializationManifest(parser_version=self.parser_version, files=next_files) self._write_manifest(next_manifest) return _materialization_result( @@ -342,7 +344,7 @@ def _materialize_full_atomic( rebuilt_graphs[path] = graph rebuilt_entries[path] = _manifest_entry(snapshot, graph) - next_manifest = MaterializationManifest(files=rebuilt_entries) + next_manifest = MaterializationManifest(parser_version=self.parser_version, files=rebuilt_entries) target_db_path = _filesystem_db_path(self.db_path) temp_db_path = _temporary_sibling(target_db_path, suffix=".lbug.tmp") temp_manifest_path = _temporary_sibling(self.manifest_path, suffix=".manifest.tmp") @@ -420,7 +422,7 @@ def _scan_source_files(self) -> tuple[dict[str, SourceSnapshot], list[str]]: if _is_excluded(path, self.source_root): continue relative_path = path.relative_to(self.source_root).as_posix() - language = SUPPORTED_SUFFIXES.get(path.suffix) + language = self.parser_registry.language_for_path(path) snapshots[relative_path] = SourceSnapshot( path=relative_path, absolute_path=path, @@ -435,7 +437,7 @@ def _build_graph(self, snapshot: SourceSnapshot) -> CodeGraph: if snapshot.language is None: raise ValueError(f"Cannot build graph for unsupported file: {snapshot.path}") try: - parser = parser_for_language(snapshot.language) + parser = self.parser_registry.parser_for_language(snapshot.language) bundle = parser.parse_file( snapshot.absolute_path, relative_path=snapshot.path, diff --git a/src/codebase_graph/ingest/tree_sitter_parser.py b/src/codebase_graph/ingest/tree_sitter_parser.py index 27873dd..95d870d 100644 --- a/src/codebase_graph/ingest/tree_sitter_parser.py +++ b/src/codebase_graph/ingest/tree_sitter_parser.py @@ -1,9 +1,10 @@ from __future__ import annotations import re +from collections.abc import Callable, Mapping from dataclasses import dataclass from pathlib import Path -from typing import Any +from typing import Any, Protocol from codebase_graph.extract import ParseBundle from .document_parser import MarkdownDocumentParser @@ -13,6 +14,71 @@ class ParserUnavailableError(RuntimeError): pass +class SourceParser(Protocol): + language: str + parser_version: str + + def parse_file( + self, + path: Path, + *, + relative_path: str, + source_root: Path, + repository_label: str, + content_hash: str, + ) -> ParseBundle: + ... + + +@dataclass(frozen=True, slots=True) +class ParserRegistration: + language: str + suffixes: tuple[str, ...] + parser_factory: Callable[[], SourceParser] + parser_version: str + + +class ParserRegistry: + def __init__(self, registrations: Mapping[str, ParserRegistration] | None = None) -> None: + self._registrations: dict[str, ParserRegistration] = dict(registrations or {}) + self._suffix_to_language: dict[str, str] = {} + for registration in self._registrations.values(): + self._register_suffixes(registration) + + @property + def parser_version(self) -> str: + return "+".join( + registration.parser_version + for registration in self._registrations.values() + ) + + def register( + self, + language: str, + *, + suffixes: tuple[str, ...], + parser_factory: Callable[[], SourceParser], + parser_version: str, + ) -> None: + registration = ParserRegistration(language, suffixes, parser_factory, parser_version) + self._registrations[language] = registration + self._register_suffixes(registration) + + def language_for_path(self, path: Path) -> str | None: + return self._suffix_to_language.get(path.suffix) + + def parser_for_language(self, language: str) -> SourceParser: + try: + registration = self._registrations[language] + except KeyError as exc: + raise ValueError(f"Unsupported materializer language: {language}") from exc + return registration.parser_factory() + + def _register_suffixes(self, registration: ParserRegistration) -> None: + for suffix in registration.suffixes: + self._suffix_to_language[suffix] = registration.language + + @dataclass(frozen=True, slots=True) class TreeSitterPythonParser: language: str = "python" @@ -45,12 +111,25 @@ def parse_source(self, source_text: str) -> dict[str, Any]: return _convert_node(tree.root_node, source_bytes) -def parser_for_language(language: str) -> TreeSitterPythonParser | MarkdownDocumentParser: - if language == "python": - return TreeSitterPythonParser() - if language == "markdown": - return MarkdownDocumentParser() - raise ValueError(f"Unsupported materializer language: {language}") +def default_parser_registry() -> ParserRegistry: + registry = ParserRegistry() + registry.register( + "python", + suffixes=(".py",), + parser_factory=TreeSitterPythonParser, + parser_version=TreeSitterPythonParser().parser_version, + ) + registry.register( + "markdown", + suffixes=(".md", ".mdx"), + parser_factory=MarkdownDocumentParser, + parser_version=MarkdownDocumentParser().parser_version, + ) + return registry + + +def parser_for_language(language: str) -> SourceParser: + return default_parser_registry().parser_for_language(language) def _python_parser() -> Any: diff --git a/src/codebase_graph/ontology/ontology.py b/src/codebase_graph/ontology/ontology.py index 6958f88..d7e9b50 100644 --- a/src/codebase_graph/ontology/ontology.py +++ b/src/codebase_graph/ontology/ontology.py @@ -687,6 +687,29 @@ def _relation( }, } +SYMBOL_LOOKUP_DEFINITION_TYPES = ( + "Class", + "Function", + "Method", + "Variable", + "Constant", + "ClassAttribute", + "InstanceAttribute", + "Property", + "Parameter", + "TypeAlias", +) + +SYMBOL_LOOKUP_QUERY = ( + " UNION ALL ".join( + f"MATCH (s:{node_type}) " + "WHERE s.label = $name OR s.qualified_name = $name " + "RETURN s.id, s.label, s.qualified_name, s.path" + for node_type in SYMBOL_LOOKUP_DEFINITION_TYPES + ) + + " LIMIT 25" +) + QUERY_HELPERS = ( QueryHelperSpec( "repository_overview", @@ -696,8 +719,8 @@ def _relation( ), QueryHelperSpec( "symbol_lookup", - "Find declarations by label or qualified name.", - "MATCH (s:Symbol) WHERE s.label = $name OR s.qualified_name = $name RETURN s.id, s.label, s.qualified_name, s.path LIMIT 25", + "Find concrete semantic definitions by label or qualified name.", + SYMBOL_LOOKUP_QUERY, parameters=("name",), returns=("id", "label", "qualified_name", "path"), ), @@ -734,7 +757,9 @@ def _relation( QueryHelperSpec( "unresolved_references", "Find references that have not been resolved to a semantic target.", - "MATCH (r:Reference) RETURN r.id, r.label, r.path, r.line_start LIMIT 100", + "MATCH (r:Reference) " + "WHERE NOT EXISTS { MATCH (r)-[:FROM_ResolvesTo]->(:ResolvesTo)-[:TO_ResolvesTo]->() } " + "RETURN r.id, r.label, r.path, r.line_start LIMIT 100", returns=("id", "label", "path", "line_start"), ), ) diff --git a/src/codebase_graph/paths.py b/src/codebase_graph/paths.py new file mode 100644 index 0000000..ed373aa --- /dev/null +++ b/src/codebase_graph/paths.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +DEFAULT_STATE_DIR = ".codebaseGraph" +CONFIG_NAME = "config.json" +MANIFEST_NAME = "manifest.json" +MCP_SERVER_NAME = "codebaseGraph" + + +@dataclass(frozen=True, slots=True) +class GraphStatePaths: + repo_root: Path + repo_name: str + state_dir: Path + db_path: Path + manifest_path: Path + config_path: Path + + def as_dict(self) -> dict[str, str]: + return { + "repo_root": self.repo_root.as_posix(), + "repo_name": self.repo_name, + "state_dir": self.state_dir.as_posix(), + "db_path": self.db_path.as_posix(), + "manifest_path": self.manifest_path.as_posix(), + "config_path": self.config_path.as_posix(), + } + + +def derive_graph_state_paths(repo_root: str | Path) -> GraphStatePaths: + root = Path(repo_root).expanduser().resolve() + repo_name = _repo_name(root) + state_dir = root / DEFAULT_STATE_DIR + return GraphStatePaths( + repo_root=root, + repo_name=repo_name, + state_dir=state_dir, + db_path=state_dir / f"{repo_name}_graph.ldb", + manifest_path=state_dir / MANIFEST_NAME, + config_path=state_dir / CONFIG_NAME, + ) + + +def _repo_name(root: Path) -> str: + name = root.name.strip() + if name: + return _safe_name(name) + return "repository" + + +def _safe_name(value: str) -> str: + normalized = "".join(character if character.isalnum() or character in {"-", "_"} else "_" for character in value) + return normalized.strip("._-") or "repository" diff --git a/src/codebase_graph/reasoning/context_builder.py b/src/codebase_graph/reasoning/context_builder.py index 9c1b3fa..ddac2df 100644 --- a/src/codebase_graph/reasoning/context_builder.py +++ b/src/codebase_graph/reasoning/context_builder.py @@ -3,7 +3,7 @@ from dataclasses import dataclass, field from typing import Any -from codebase_graph.db.schema import quote_identifier +from codebase_graph.db import graph_query_adapter from codebase_graph.ontology import CONTEXT_PROFILES, RELATION_TYPES @@ -37,6 +37,7 @@ def as_dict(self) -> dict[str, Any]: class CompactContextBuilder: def __init__(self, store: Any) -> None: self.store = store + self.query = graph_query_adapter(store) self._relation_names = {relation_type.name for relation_type in RELATION_TYPES} def build( @@ -113,35 +114,24 @@ def _query_neighbors( direction: str, limit: int, ) -> list[ContextNode]: - if direction == "outgoing": - statement = ( - f"MATCH (source:{quote_identifier(node_type)} {{id: $node_id}})" - f"-[:{quote_identifier(f'FROM_{relation}')}]->(edge:{quote_identifier(relation)})" - f"-[:{quote_identifier(f'TO_{relation}')}]->(neighbor) " - "RETURN neighbor.id, neighbor.label, neighbor.qualified_name, neighbor.path, " - f"neighbor.line_start, neighbor.line_end, neighbor.summary LIMIT {int(limit)}" - ) - else: - statement = ( - "MATCH (neighbor)" - f"-[:{quote_identifier(f'FROM_{relation}')}]->(edge:{quote_identifier(relation)})" - f"-[:{quote_identifier(f'TO_{relation}')}]->(target:{quote_identifier(node_type)} {{id: $node_id}}) " - "RETURN neighbor.id, neighbor.label, neighbor.qualified_name, neighbor.path, " - f"neighbor.line_start, neighbor.line_end, neighbor.summary LIMIT {int(limit)}" - ) - rows = self.store.execute(statement, {"node_id": node_id}).get_all() return [ ContextNode( relation=relation, direction=direction, - type=_type_from_id(_value(row, 0)), - label=_text(_value(row, 1)) or _text(_value(row, 2)), - path=_text(_value(row, 3)), - span=_span(_value(row, 4), _value(row, 5)), - summary=_text(_value(row, 6)), - id=_text(_value(row, 0)), + type=neighbor.node_type, + label=neighbor.label or neighbor.qualified_name, + path=neighbor.path, + span=_span(neighbor.line_start, neighbor.line_end), + summary=neighbor.summary, + id=neighbor.node_id, + ) + for neighbor in self.query.neighbors( + node_id=node_id, + node_type=node_type, + relation=relation, + direction=direction, + limit=limit, ) - for row in rows ] @@ -177,13 +167,6 @@ def _node_key(node: ContextNode) -> str: return node.id -def _type_from_id(value: Any) -> str: - text = _text(value) - if ":" not in text: - return "" - return text.split(":", 1)[0] - - def _span(line_start: Any, line_end: Any) -> dict[str, int]: span: dict[str, int] = {} if line_start is not None: @@ -193,15 +176,4 @@ def _span(line_start: Any, line_end: Any) -> dict[str, int]: return span -def _text(value: Any) -> str: - return "" if value is None else str(value) - - -def _value(row: Any, index: int) -> Any: - try: - return row[index] - except IndexError: - return None - - __all__ = ["CompactContextBuilder", "ContextNode", "DEFAULT_CONTEXT_BUDGET", "DEFAULT_CONTEXT_LIMIT"] diff --git a/src/codebase_graph/retrieval/search.py b/src/codebase_graph/retrieval/search.py index 396c9ee..e6724e2 100644 --- a/src/codebase_graph/retrieval/search.py +++ b/src/codebase_graph/retrieval/search.py @@ -3,6 +3,7 @@ from dataclasses import dataclass, field from typing import Any +from codebase_graph.db import SearchIndexRow, graph_query_adapter from codebase_graph.ontology import CONTEXT_PROFILES, SEARCH_INDEXES from codebase_graph.reasoning.context_builder import CompactContextBuilder, ContextNode, DEFAULT_CONTEXT_BUDGET, DEFAULT_CONTEXT_LIMIT @@ -95,6 +96,7 @@ class FTSIndexSpec: class SearchService: def __init__(self, store: Any) -> None: self.store = store + self.query = graph_query_adapter(store) self.indexes = tuple(_fts_index_specs()) def search(self, request: SearchRequest) -> CompactContextPayload: @@ -128,12 +130,15 @@ def search(self, request: SearchRequest) -> CompactContextPayload: def _query_fts(self, query: str, limit: int) -> list[SearchHit]: hits: list[SearchHit] = [] for spec in self.indexes: - result = self.store.execute( - _fts_query_statement(spec), - {"query": query, "top": limit}, + hits.extend( + _hit_from_index_row(row, spec) + for row in self.query.search_index( + node_type=spec.node_type, + index_name=spec.index_name, + query=query, + limit=limit, + ) ) - rows = result.get_all() - hits.extend(_hit_from_row(row, spec) for row in rows) return hits def _rank_hits(self, hits: list[SearchHit], *, query: str = "", profile: str = "brief") -> list[SearchHit]: @@ -147,14 +152,6 @@ def _rank_hits(self, hits: list[SearchHit], *, query: str = "", profile: str = " return sorted(deduped, key=_ranked_hit_sort_key) -def _fts_query_statement(spec: FTSIndexSpec) -> str: - return ( - f"CALL QUERY_FTS_INDEX('{spec.node_type}', '{spec.index_name}', $query, TOP := $top) " - "RETURN node.id, node.label, node.qualified_name, node.path, " - "node.line_start, node.line_end, node.summary, score" - ) - - def _fts_index_specs() -> list[FTSIndexSpec]: specs: list[FTSIndexSpec] = [] order = 0 @@ -166,16 +163,16 @@ def _fts_index_specs() -> list[FTSIndexSpec]: return specs -def _hit_from_row(row: Any, spec: FTSIndexSpec) -> SearchHit: +def _hit_from_index_row(row: SearchIndexRow, spec: FTSIndexSpec) -> SearchHit: return SearchHit( - id=_text(_value(row, 0)), + id=row.id, type=spec.node_type, - label=_text(_value(row, 1)), - qualified_name=_text(_value(row, 2)), - path=_text(_value(row, 3)), - span=_span(_value(row, 4), _value(row, 5)), - summary=_text(_value(row, 6)), - score=float(_value(row, 7) or 0.0), + label=row.label, + qualified_name=row.qualified_name, + path=row.path, + span=_span(row.line_start, row.line_end), + summary=row.summary, + score=row.score, index_order=spec.order, ) @@ -304,17 +301,6 @@ def _span(line_start: Any, line_end: Any) -> dict[str, int]: return span -def _text(value: Any) -> str: - return "" if value is None else str(value) - - -def _value(row: Any, index: int) -> Any: - try: - return row[index] - except IndexError: - return None - - __all__ = [ "CompactContextPayload", "DEFAULT_SEARCH_LIMIT", diff --git a/src/codebase_graph/setup/__init__.py b/src/codebase_graph/setup/__init__.py index 02b4e36..77337e4 100644 --- a/src/codebase_graph/setup/__init__.py +++ b/src/codebase_graph/setup/__init__.py @@ -4,8 +4,10 @@ from .state import ( CONFIG_NAME, DEFAULT_STATE_DIR, + GraphStatePaths, MANIFEST_NAME, SetupPaths, + derive_graph_state_paths, derive_setup_paths, load_setup_config, ) @@ -13,11 +15,13 @@ __all__ = [ "CONFIG_NAME", "DEFAULT_STATE_DIR", + "GraphStatePaths", "MANIFEST_NAME", "SetupError", "SetupOptions", "SetupPaths", "SetupResult", + "derive_graph_state_paths", "derive_setup_paths", "load_setup_config", "run_setup", diff --git a/src/codebase_graph/setup/state.py b/src/codebase_graph/setup/state.py index 2e0b67c..1cac1d5 100644 --- a/src/codebase_graph/setup/state.py +++ b/src/codebase_graph/setup/state.py @@ -2,55 +2,29 @@ import json import os -from dataclasses import dataclass from importlib.metadata import PackageNotFoundError, version from pathlib import Path from typing import Any +from codebase_graph import paths as graph_paths from codebase_graph.ontology import ONTOLOGY_VERSION -DEFAULT_STATE_DIR = ".codebaseGraph" -CONFIG_NAME = "config.json" -MANIFEST_NAME = "manifest.json" -MCP_SERVER_NAME = "codebaseGraph" - - -@dataclass(frozen=True, slots=True) -class SetupPaths: - repo_root: Path - repo_name: str - state_dir: Path - db_path: Path - manifest_path: Path - config_path: Path - - def as_dict(self) -> dict[str, str]: - return { - "repo_root": self.repo_root.as_posix(), - "repo_name": self.repo_name, - "state_dir": self.state_dir.as_posix(), - "db_path": self.db_path.as_posix(), - "manifest_path": self.manifest_path.as_posix(), - "config_path": self.config_path.as_posix(), - } +CONFIG_NAME = graph_paths.CONFIG_NAME +DEFAULT_STATE_DIR = graph_paths.DEFAULT_STATE_DIR +MANIFEST_NAME = graph_paths.MANIFEST_NAME +MCP_SERVER_NAME = graph_paths.MCP_SERVER_NAME +GraphStatePaths = graph_paths.GraphStatePaths +derive_graph_state_paths = graph_paths.derive_graph_state_paths +SetupPaths = graph_paths.GraphStatePaths def derive_setup_paths(repo_root: str | Path) -> SetupPaths: - root = Path(repo_root).expanduser().resolve() - if not root.exists(): - raise FileNotFoundError(f"Repository root does not exist: {root}") - if not root.is_dir(): - raise NotADirectoryError(f"Repository root is not a directory: {root}") - repo_name = _repo_name(root) - state_dir = root / DEFAULT_STATE_DIR - return SetupPaths( - repo_root=root, - repo_name=repo_name, - state_dir=state_dir, - db_path=state_dir / f"{repo_name}_graph.ldb", - manifest_path=state_dir / MANIFEST_NAME, - config_path=state_dir / CONFIG_NAME, - ) + paths = derive_graph_state_paths(repo_root) + if not paths.repo_root.exists(): + raise FileNotFoundError(f"Repository root does not exist: {paths.repo_root}") + if not paths.repo_root.is_dir(): + raise NotADirectoryError(f"Repository root is not a directory: {paths.repo_root}") + return paths def build_setup_config(paths: SetupPaths, *, mcp_command: list[str]) -> dict[str, Any]: @@ -94,18 +68,6 @@ def write_setup_config(path: Path, payload: dict[str, Any]) -> str: return action -def _repo_name(root: Path) -> str: - name = root.name.strip() - if name: - return _safe_name(name) - return "repository" - - -def _safe_name(value: str) -> str: - normalized = "".join(character if character.isalnum() or character in {"-", "_"} else "_" for character in value) - return normalized.strip("._-") or "repository" - - def _package_version() -> str: try: return version("codebase-graph") diff --git a/tests/test_graph_builder.py b/tests/test_graph_builder.py index 01c66bb..a474f6b 100644 --- a/tests/test_graph_builder.py +++ b/tests/test_graph_builder.py @@ -1,6 +1,6 @@ from __future__ import annotations -from codebase_graph.extract import CaptureRecord, GraphBuilder, ParseBundle +from codebase_graph.extract import CaptureRecord, CaptureTableRegistry, GraphBuilder, ParseBundle from codebase_graph.ontology import PARSER_NODE_MAPPINGS @@ -119,6 +119,25 @@ def test_graph_builder_uses_capture_names_as_primary_semantic_signal() -> None: assert not result.unresolved +def test_graph_builder_accepts_registered_capture_table_mapping() -> None: + registry = CaptureTableRegistry() + registry.register_exact("custom.component", "Component") + bundle = ParseBundle( + language="custom", + path="component.custom", + captures=( + CaptureRecord( + "custom.component", + {"type": "identifier", "name": "RegisteredWidget", "line_start": 1}, + ), + ), + ) + + result = GraphBuilder(capture_table_registry=registry).build_file_graph(bundle) + + assert {node.label for node in result.graph.nodes_by_type("Component")} == {"RegisteredWidget"} + + def test_graph_builder_routes_local_imports_through_containing_scope() -> None: parse_tree = { "type": "Module", diff --git a/tests/test_materializer.py b/tests/test_materializer.py index bf8c20c..96b55c5 100644 --- a/tests/test_materializer.py +++ b/tests/test_materializer.py @@ -9,8 +9,10 @@ from codebase_graph.db import LadybugCodeGraphStore from codebase_graph.ingest import ( GraphMaterializer, + MarkdownDocumentParser, ManifestEntry, MaterializationManifest, + ParserRegistry, SourceSnapshot, TreeSitterPythonParser, ) @@ -72,6 +74,36 @@ def test_tree_sitter_python_parser_maps_sample_fixture_to_graph_tree() -> None: assert any(child["type"] == "function_definition" and child["name"] == "helper" for child in bundle.tree["children"]) +def test_materializer_defaults_to_canonical_codebasegraph_state_paths(tmp_path: Path) -> None: + source_root = tmp_path / "sample repo" + source_root.mkdir() + + materializer = GraphMaterializer(source_root, store=object()) + + assert materializer.state_dir == source_root / ".codebaseGraph" + assert materializer.db_path == source_root / ".codebaseGraph" / "sample_repo_graph.ldb" + assert materializer.manifest_path == source_root / ".codebaseGraph" / "manifest.json" + + +def test_scan_source_files_uses_parser_registry_for_suffix_mapping(tmp_path: Path) -> None: + registry = ParserRegistry() + registry.register( + "notes", + suffixes=(".notes",), + parser_factory=MarkdownDocumentParser, + parser_version="notes-v1", + ) + source_root = tmp_path / "project" + source_root.mkdir() + (source_root / "handoff.notes").write_text("# Handoff\n", encoding="utf-8") + + materializer = GraphMaterializer(source_root, store=object(), parser_registry=registry) + snapshots, diagnostics = materializer._scan_source_files() + + assert snapshots["handoff.notes"].language == "notes" + assert not diagnostics + + def test_scan_source_files_prunes_excluded_directories(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: source_dir = tmp_path / "src" source_dir.mkdir() diff --git a/tests/test_ontology.py b/tests/test_ontology.py index 06c4fd0..7ea6e6a 100644 --- a/tests/test_ontology.py +++ b/tests/test_ontology.py @@ -133,6 +133,21 @@ def test_query_helpers_use_edge_node_relation_traversal() -> None: assert "[:TO_Documents]" in helper_queries["documentation_context"] +def test_query_helper_semantics_match_public_names() -> None: + helper_queries = {helper.name: helper.query for helper in QUERY_HELPERS} + + symbol_lookup = helper_queries["symbol_lookup"] + assert ":Class" in symbol_lookup + assert ":Function" in symbol_lookup + assert ":Method" in symbol_lookup + assert "s:Symbol" not in symbol_lookup + + unresolved_references = helper_queries["unresolved_references"] + assert "NOT EXISTS" in unresolved_references + assert "FROM_ResolvesTo" in unresolved_references + assert "TO_ResolvesTo" in unresolved_references + + def test_lookup_helpers_return_expected_specs() -> None: assert get_node_type("Class").name == "Class" assert get_relation_type("Calls").name == "Calls" diff --git a/tests/test_search.py b/tests/test_search.py index 93415bf..528cc69 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -8,6 +8,7 @@ import pytest from codebase_graph.cli import main as cli_main +from codebase_graph.db import GraphNeighbor, SearchIndexRow from codebase_graph.ingest import GraphMaterializer from codebase_graph.reasoning import CompactContextBuilder from codebase_graph.retrieval.search import SearchHit, SearchRequest, SearchService @@ -31,6 +32,64 @@ def execute(self, statement: str, parameters: dict[str, Any] | None = None) -> _ return _Result(self.rows) +class _Adapter: + def __init__(self) -> None: + self.search_calls: list[dict[str, Any]] = [] + self.neighbor_calls: list[dict[str, Any]] = [] + + def search_index(self, *, node_type: str, index_name: str, query: str, limit: int) -> list[SearchIndexRow]: + self.search_calls.append({"node_type": node_type, "index_name": index_name, "query": query, "limit": limit}) + if node_type != "Class": + return [] + return [ + SearchIndexRow( + id="opaque-class-id", + node_type="Class", + label="SampleService", + qualified_name="sample.SampleService", + path="sample/service.py", + score=1.0, + ) + ] + + def neighbors( + self, + *, + node_id: str, + node_type: str, + relation: str, + direction: str, + limit: int, + ) -> list[GraphNeighbor]: + self.neighbor_calls.append( + { + "node_id": node_id, + "node_type": node_type, + "relation": relation, + "direction": direction, + "limit": limit, + } + ) + if relation != "Defines" or direction != "outgoing": + return [] + return [ + GraphNeighbor( + node_id="opaque-neighbor-id", + node_type="Method", + label="run", + path="sample/service.py", + line_start=2, + line_end=3, + summary="Run the service.", + ) + ] + + +class _AdapterStore: + def __init__(self, adapter: _Adapter) -> None: + self.graph_query_adapter = adapter + + def test_search_query_uses_ontology_index_names_and_parameterized_user_text() -> None: malicious_query = "SampleService'); MATCH (n) RETURN n" store = _RecordingStore() @@ -159,6 +218,29 @@ def test_compact_context_respects_max_depth_limit_and_budget() -> None: assert len(context[0].summary) < len(long_summary) +def test_compact_context_uses_adapter_types_and_opaque_node_ids() -> None: + adapter = _Adapter() + builder = CompactContextBuilder(_AdapterStore(adapter)) + + context = builder.build("opaque-class-id", "Class", profile="definitions", limit=1, budget=120, max_depth=1) + + assert context[0].id == "opaque-neighbor-id" + assert context[0].type == "Method" + assert context[0].label == "run" + assert adapter.neighbor_calls[0]["node_id"] == "opaque-class-id" + + +def test_search_service_uses_query_adapter_for_fts() -> None: + adapter = _Adapter() + + payload = SearchService(_AdapterStore(adapter)).search(SearchRequest("SampleService", limit=1, budget=0)) + + data = payload.as_dict() + assert data["results"][0]["id"] == "opaque-class-id" + assert data["results"][0]["type"] == "Class" + assert adapter.search_calls + + def test_search_request_rejects_invalid_profile() -> None: with pytest.raises(ValueError, match="Unknown context profile"): SearchRequest("SampleService", profile="missing").validate() From f74a1dfedda273e62b48a2330d5eb0adb4668827 Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Mon, 25 May 2026 11:40:57 +0930 Subject: [PATCH 12/53] fix: improve graph setup --- .gitignore | 4 ++++ AGENTS.md | 7 +++++++ README.md | 2 +- src/codebase_graph/setup/instructions.py | 7 ++++--- src/codebase_graph/setup/mcp_config.py | 11 ++++++++++- src/codebase_graph/setup/orchestrator.py | 1 + tests/test_setup_workflow.py | 19 ++++++++++++++++++- 7 files changed, 45 insertions(+), 6 deletions(-) create mode 100644 AGENTS.md diff --git a/.gitignore b/.gitignore index 83972fa..ea0776e 100644 --- a/.gitignore +++ b/.gitignore @@ -206,6 +206,10 @@ tempCodeRunnerFile.py # Ruff stuff: .ruff_cache/ +# codebaseGraph local graph state +.codebaseGraph/ +.codebase_graph/ + # PyPI configuration file .pypirc diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..9fd59f2 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,7 @@ + +## codebaseGraph workflow +- Use the `codebaseGraph` MCP server for repository graph search, schema, and compact context before answering repo-structure questions. +- Prefer `graph_search` for symbols, paths, docs, and setup instructions; follow with `graph_context` when relationships or nearby evidence matter. +- Use `graph_schema` or `graph_query_helpers` before writing raw graph queries, and keep `graph_query` read-only. +- Refresh the graph with `/Users/rabii/Projects/Repositories/codebaseGraph/.venv/bin/codebase-graph setup --repo-root .` when files change materially. Setup config: `/Users/rabii/Projects/Repositories/codebaseGraph/.codebaseGraph/config.json`. + diff --git a/README.md b/README.md index 061467b..ef6ff89 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ codebase-graph setup --instructions-target claude ## MCP usage -Setup writes an MCP server entry equivalent to: +Setup writes an MCP server entry equivalent to the block below. When setup is run from a virtual environment, the command may be the absolute path to that environment's `codebase-graph` executable so the MCP client can launch it without relying on shell `PATH`. ```json { diff --git a/src/codebase_graph/setup/instructions.py b/src/codebase_graph/setup/instructions.py index 9f0fb61..310d532 100644 --- a/src/codebase_graph/setup/instructions.py +++ b/src/codebase_graph/setup/instructions.py @@ -22,12 +22,13 @@ def upsert_instruction_block( target: str = "auto", server_name: str, config_path: Path, + setup_command: str = "codebase-graph", ) -> InstructionResult: if target == "skip": return InstructionResult("skipped", None) path = _select_instruction_path(repo_root, target) existing = path.read_text(encoding="utf-8") if path.exists() else "" - block = _instruction_block(server_name=server_name, config_path=config_path) + block = _instruction_block(server_name=server_name, config_path=config_path, setup_command=setup_command) next_text, action = _upsert_block(existing, block, created=not path.exists()) if next_text == existing: return InstructionResult("unchanged", path.as_posix()) @@ -65,14 +66,14 @@ def _select_instruction_path(repo_root: Path, target: str) -> Path: return agents -def _instruction_block(*, server_name: str, config_path: Path) -> str: +def _instruction_block(*, server_name: str, config_path: Path, setup_command: str) -> str: return ( f"{START_MARKER}\n" "## codebaseGraph workflow\n" f"- Use the `{server_name}` MCP server for repository graph search, schema, and compact context before answering repo-structure questions.\n" "- Prefer `graph_search` for symbols, paths, docs, and setup instructions; follow with `graph_context` when relationships or nearby evidence matter.\n" "- Use `graph_schema` or `graph_query_helpers` before writing raw graph queries, and keep `graph_query` read-only.\n" - f"- Refresh the graph with `codebase-graph setup --repo-root .` when files change materially. Setup config: `{config_path.as_posix()}`.\n" + f"- Refresh the graph with `{setup_command} setup --repo-root .` when files change materially. Setup config: `{config_path.as_posix()}`.\n" f"{END_MARKER}\n" ) diff --git a/src/codebase_graph/setup/mcp_config.py b/src/codebase_graph/setup/mcp_config.py index 87d4432..671a1a0 100644 --- a/src/codebase_graph/setup/mcp_config.py +++ b/src/codebase_graph/setup/mcp_config.py @@ -2,6 +2,8 @@ import json import os +import shutil +import sys from dataclasses import dataclass from pathlib import Path from typing import Any @@ -53,7 +55,7 @@ def configure_mcp_client( def server_entry(setup_config_path: Path) -> dict[str, Any]: return { - "command": "codebase-graph", + "command": _resolve_server_command(), "args": ["mcp", "serve", "--config", setup_config_path.as_posix()], } @@ -70,6 +72,13 @@ def default_config_path(client: str) -> Path: raise ValueError(f"Unsupported MCP client: {client}") +def _resolve_server_command() -> str: + sibling_script = Path(sys.executable).with_name("codebase-graph") + if sibling_script.exists() and os.access(sibling_script, os.X_OK): + return sibling_script.as_posix() + return shutil.which("codebase-graph") or "codebase-graph" + + def _next_config_payload(path: Path, entry: dict[str, Any]) -> tuple[dict[str, Any], str]: payload = _read_json(path) servers = payload.setdefault("mcpServers", {}) diff --git a/src/codebase_graph/setup/orchestrator.py b/src/codebase_graph/setup/orchestrator.py index e431c46..1b22bab 100644 --- a/src/codebase_graph/setup/orchestrator.py +++ b/src/codebase_graph/setup/orchestrator.py @@ -60,6 +60,7 @@ def run_setup(options: SetupOptions) -> SetupResult: target=options.instructions_target, server_name=MCP_SERVER_NAME, config_path=paths.config_path, + setup_command=mcp_entry["command"], ) materializer = GraphMaterializer( paths.repo_root, diff --git a/tests/test_setup_workflow.py b/tests/test_setup_workflow.py index 278e8d0..13cc08d 100644 --- a/tests/test_setup_workflow.py +++ b/tests/test_setup_workflow.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import sys from pathlib import Path try: @@ -15,7 +16,7 @@ from codebase_graph.mcp.server import McpGraphServer, handle_tool_call from codebase_graph.setup import SetupError, SetupOptions, run_setup from codebase_graph.setup.instructions import END_MARKER, START_MARKER -from codebase_graph.setup.mcp_config import configure_mcp_client +from codebase_graph.setup.mcp_config import configure_mcp_client, server_entry def test_setup_cli_creates_state_db_mcp_config_instructions_and_searchable_docs( @@ -124,6 +125,22 @@ def test_mcp_config_dry_run_preserves_existing_servers(tmp_path: Path) -> None: assert set(payload["mcpServers"]) == {"otherServer", "codebaseGraph"} +def test_server_entry_prefers_current_environment_script(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + bin_dir = tmp_path / "venv" / "bin" + bin_dir.mkdir(parents=True) + python_path = bin_dir / "python" + script_path = bin_dir / "codebase-graph" + python_path.write_text("", encoding="utf-8") + script_path.write_text("", encoding="utf-8") + script_path.chmod(0o755) + monkeypatch.setattr(sys, "executable", python_path.as_posix()) + monkeypatch.setenv("PATH", "") + + entry = server_entry(tmp_path / ".codebaseGraph" / "config.json") + + assert entry["command"] == script_path.as_posix() + + def test_setup_preflight_failure_stops_before_state_creation(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: repo_root = _fresh_repo(tmp_path) From 415023946da4b10fa09147aecafbb0e54d2af371 Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Mon, 25 May 2026 11:43:20 +0930 Subject: [PATCH 13/53] fix: clear workflow instructions --- AGENTS.md | 2 +- src/codebase_graph/setup/instructions.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 9fd59f2..f270814 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,6 +1,6 @@ ## codebaseGraph workflow -- Use the `codebaseGraph` MCP server for repository graph search, schema, and compact context before answering repo-structure questions. +- Use the `codebaseGraph` MCP server for repository graph search, schema, and compact context before answering repo-structure questions or performing coding tasks. - Prefer `graph_search` for symbols, paths, docs, and setup instructions; follow with `graph_context` when relationships or nearby evidence matter. - Use `graph_schema` or `graph_query_helpers` before writing raw graph queries, and keep `graph_query` read-only. - Refresh the graph with `/Users/rabii/Projects/Repositories/codebaseGraph/.venv/bin/codebase-graph setup --repo-root .` when files change materially. Setup config: `/Users/rabii/Projects/Repositories/codebaseGraph/.codebaseGraph/config.json`. diff --git a/src/codebase_graph/setup/instructions.py b/src/codebase_graph/setup/instructions.py index 310d532..2c327ef 100644 --- a/src/codebase_graph/setup/instructions.py +++ b/src/codebase_graph/setup/instructions.py @@ -70,7 +70,7 @@ def _instruction_block(*, server_name: str, config_path: Path, setup_command: st return ( f"{START_MARKER}\n" "## codebaseGraph workflow\n" - f"- Use the `{server_name}` MCP server for repository graph search, schema, and compact context before answering repo-structure questions.\n" + f"- Use the `{server_name}` MCP server for repository graph search, schema, and compact context before answering repo-structure questions or performing coding tasks.\n" "- Prefer `graph_search` for symbols, paths, docs, and setup instructions; follow with `graph_context` when relationships or nearby evidence matter.\n" "- Use `graph_schema` or `graph_query_helpers` before writing raw graph queries, and keep `graph_query` read-only.\n" f"- Refresh the graph with `{setup_command} setup --repo-root .` when files change materially. Setup config: `{config_path.as_posix()}`.\n" From 936f959d5415efe1d9f5f805d9edc21a6955d036 Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Mon, 25 May 2026 13:17:37 +0930 Subject: [PATCH 14/53] feat: transport layers for MCP server - Added runtime configuration management in `runtime.py` to handle repository and database paths. - Implemented tool handling and execution logic in `tools.py`, supporting various graph-related operations. - Created transport mechanisms for HTTP and standard input/output in `transports/http.py` and `transports/stdio.py`. - Developed client adapters for different configurations in `clients` directory, including Codex, Hermes, and JSON-based clients. - Established a server descriptor to manage server configurations and command execution in `descriptor.py`. - Added comprehensive tests for MCP portability and functionality in `test_mcp_portability.py`. --- AGENTS.md | 3 +- README.md | 54 ++- src/codebase_graph/cli/__init__.py | 44 ++- src/codebase_graph/db/store.py | 32 +- src/codebase_graph/ingest/materializer.py | 11 + src/codebase_graph/mcp/__init__.py | 4 +- src/codebase_graph/mcp/protocol.py | 98 +++++ src/codebase_graph/mcp/runtime.py | 52 +++ src/codebase_graph/mcp/server.py | 353 +----------------- src/codebase_graph/mcp/tools.py | 228 +++++++++++ src/codebase_graph/mcp/transports/__init__.py | 1 + src/codebase_graph/mcp/transports/http.py | 142 +++++++ src/codebase_graph/mcp/transports/stdio.py | 52 +++ src/codebase_graph/setup/clients/__init__.py | 34 ++ src/codebase_graph/setup/clients/base.py | 34 ++ src/codebase_graph/setup/clients/codex.py | 81 ++++ src/codebase_graph/setup/clients/hermes.py | 71 ++++ .../setup/clients/json_clients.py | 95 +++++ src/codebase_graph/setup/descriptor.py | 81 ++++ src/codebase_graph/setup/instructions.py | 3 +- src/codebase_graph/setup/mcp_config.py | 98 +++-- src/codebase_graph/setup/orchestrator.py | 5 +- tests/test_materializer.py | 17 + tests/test_mcp_portability.py | 239 ++++++++++++ tests/test_schema.py | 16 + tests/test_setup_workflow.py | 14 +- 26 files changed, 1442 insertions(+), 420 deletions(-) create mode 100644 src/codebase_graph/mcp/protocol.py create mode 100644 src/codebase_graph/mcp/runtime.py create mode 100644 src/codebase_graph/mcp/tools.py create mode 100644 src/codebase_graph/mcp/transports/__init__.py create mode 100644 src/codebase_graph/mcp/transports/http.py create mode 100644 src/codebase_graph/mcp/transports/stdio.py create mode 100644 src/codebase_graph/setup/clients/__init__.py create mode 100644 src/codebase_graph/setup/clients/base.py create mode 100644 src/codebase_graph/setup/clients/codex.py create mode 100644 src/codebase_graph/setup/clients/hermes.py create mode 100644 src/codebase_graph/setup/clients/json_clients.py create mode 100644 src/codebase_graph/setup/descriptor.py create mode 100644 tests/test_mcp_portability.py diff --git a/AGENTS.md b/AGENTS.md index f270814..d07a77a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,7 +1,8 @@ ## codebaseGraph workflow +- Treat the `codebaseGraph` MCP server knowledge graph as the project operating source of truth. - Use the `codebaseGraph` MCP server for repository graph search, schema, and compact context before answering repo-structure questions or performing coding tasks. - Prefer `graph_search` for symbols, paths, docs, and setup instructions; follow with `graph_context` when relationships or nearby evidence matter. - Use `graph_schema` or `graph_query_helpers` before writing raw graph queries, and keep `graph_query` read-only. -- Refresh the graph with `/Users/rabii/Projects/Repositories/codebaseGraph/.venv/bin/codebase-graph setup --repo-root .` when files change materially. Setup config: `/Users/rabii/Projects/Repositories/codebaseGraph/.codebaseGraph/config.json`. +- Refresh the graph with `/Users/rabii/Projects/Repositories/codebaseGraph/.venv/bin/codebase-graph setup --repo-root . --mcp-client codex` when files change materially. Setup config: `/Users/rabii/Projects/Repositories/codebaseGraph/.codebaseGraph/config.json`. diff --git a/README.md b/README.md index ef6ff89..760c20c 100644 --- a/README.md +++ b/README.md @@ -31,29 +31,47 @@ The setup command also: - Materializes the repository graph into the repo-local database. - Writes or updates one marked codebaseGraph block in `AGENTS.md` or `CLAUDE.md`. -- Writes a Codex or Claude-compatible MCP JSON config entry named `codebaseGraph`, unless skipped. +- Writes a native MCP client config entry named `codebaseGraph`, unless skipped. Useful options: ```bash codebase-graph setup --repo-root /path/to/repo +codebase-graph setup --mcp-client codex codebase-graph setup --mcp-client claude -codebase-graph setup --mcp-config-path /tmp/mcp.json +codebase-graph setup --mcp-client claude-project +codebase-graph setup --mcp-client lmstudio +codebase-graph setup --mcp-client hermes +codebase-graph setup --mcp-client openclaw +codebase-graph setup --mcp-client generic +codebase-graph setup --mcp-config-path /tmp/client-config codebase-graph setup --dry-run codebase-graph setup --skip-mcp-config codebase-graph setup --instructions-target claude ``` -`--dry-run` returns the MCP config patch without writing the MCP client file. Repository graph state and instruction handling still run so the graph can be verified. +`--dry-run` returns the raw server descriptor plus the exact client patch or payload without writing the MCP client file. Repository graph state and instruction handling still run so the graph can be verified. ## MCP usage -Setup writes an MCP server entry equivalent to the block below. When setup is run from a virtual environment, the command may be the absolute path to that environment's `codebase-graph` executable so the MCP client can launch it without relying on shell `PATH`. +Setup builds one canonical server descriptor and serializes it into the selected client format. When setup is run from a virtual environment, the command may be the absolute path to that environment's `codebase-graph` executable so the MCP client can launch it without relying on shell `PATH`. + +Codex uses `~/.codex/config.toml`: + +```toml +[mcp_servers.codebaseGraph] +command = "codebase-graph" +args = ["mcp", "serve", "--config", ".codebaseGraph/config.json"] +startup_timeout_sec = 60 +``` + +Claude Desktop, Claude project config, LM Studio, and generic MCP JSON use an `mcpServers` shape: ```json { "mcpServers": { "codebaseGraph": { + "type": "stdio", "command": "codebase-graph", "args": ["mcp", "serve", "--config", ".codebaseGraph/config.json"] } @@ -61,6 +79,20 @@ Setup writes an MCP server entry equivalent to the block below. When setup is ru } ``` +OpenClaw uses JSON5-compatible JSON under `mcp.servers`, and Hermes emits YAML under `mcp_servers`. Use `--dry-run --mcp-client ` to inspect the exact emitted patch before writing a config file. + +Client examples: + +```bash +codebase-graph setup --repo-root . --mcp-client codex +codebase-graph setup --repo-root . --mcp-client claude +codebase-graph setup --repo-root . --mcp-client claude-project +codebase-graph setup --repo-root . --mcp-client lmstudio +codebase-graph setup --repo-root . --mcp-client hermes +codebase-graph setup --repo-root . --mcp-client openclaw +codebase-graph setup --repo-root . --mcp-client generic --dry-run +``` + The server can also be run directly: ```bash @@ -68,6 +100,14 @@ codebase-graph mcp serve --config .codebaseGraph/config.json codebase-graph-mcp --config .codebaseGraph/config.json ``` +Stdio is the default transport for local MCP clients. An optional local Streamable HTTP transport is available for clients that connect to an HTTP endpoint: + +```bash +codebase-graph mcp http --config .codebaseGraph/config.json --host 127.0.0.1 --port 8765 +``` + +Keep HTTP bound to `127.0.0.1` unless you have added authentication and understand the DNS rebinding risk for local MCP servers. + Available MCP tools: - `graph_health` @@ -105,6 +145,10 @@ ruff check . - Missing LadyBugDB: install a package build that includes `real_ladybug`; setup will fail before creating `.codebaseGraph`. - Stale graph: rerun `codebase-graph setup --repo-root .` after material source or documentation changes. -- Broken MCP config: rerun setup with `--mcp-config-path` pointing at the client JSON file, or use `--dry-run` to inspect the server block. +- Broken Codex config: rerun setup with `--mcp-client codex`, then check `codex mcp list`. +- Broken Claude config: rerun setup with `--mcp-client claude` for desktop config or `--mcp-client claude-project` for a repo-local `.mcp.json`. +- Broken LM Studio, Hermes, OpenClaw, or generic config: run setup with the matching `--mcp-client` and `--dry-run` first, then copy or write the emitted payload to the client path. +- PATH or executable issues: run setup from the virtual environment that contains `codebase-graph`; the descriptor prefers that absolute executable path. +- Direct smoke test: run `codebase-graph mcp serve --config .codebaseGraph/config.json` and send MCP `initialize`, `tools/list`, and `tools/call` JSON-RPC messages over stdio. - Unsupported files: binary, vendor, cache, virtualenv, build, dist, `.codebase_graph`, and `.codebaseGraph` paths are skipped. - Lock/contention errors: stop other graph materialization or MCP processes using the same `.codebaseGraph/_graph.ldb`, then rerun setup. diff --git a/src/codebase_graph/cli/__init__.py b/src/codebase_graph/cli/__init__.py index 85f50ad..b9c1dfd 100644 --- a/src/codebase_graph/cli/__init__.py +++ b/src/codebase_graph/cli/__init__.py @@ -5,10 +5,12 @@ from collections.abc import Sequence from pathlib import Path +from codebase_graph.db import create_ladybug_database from codebase_graph.ingest import GraphMaterializer from codebase_graph.ontology import CONTEXT_PROFILES from codebase_graph.retrieval import SearchRequest, SearchService from codebase_graph.setup import SetupError, SetupOptions, run_setup +from codebase_graph.setup.clients import supported_client_ids def main(argv: Sequence[str] | None = None) -> int: @@ -30,8 +32,8 @@ def main(argv: Sequence[str] | None = None) -> int: setup_parser = subparsers.add_parser("setup", help="Bootstrap codebaseGraph state for a repository") setup_parser.add_argument("--repo-root", default=".", help="Repository root to configure") - setup_parser.add_argument("--mcp-client", choices=("codex", "claude", "none"), default="codex") - setup_parser.add_argument("--mcp-config-path", default=None, help="Override MCP JSON config path") + setup_parser.add_argument("--mcp-client", choices=supported_client_ids(), default="codex") + setup_parser.add_argument("--mcp-config-path", default=None, help="Override MCP client config path") setup_parser.add_argument("--skip-mcp-config", action="store_true", help="Do not write MCP client config") setup_parser.add_argument("--dry-run", action="store_true", help="Return the MCP config patch without writing it") setup_parser.add_argument( @@ -50,6 +52,14 @@ def main(argv: Sequence[str] | None = None) -> int: serve_parser.add_argument("--config", default=None, help="Path to .codebaseGraph/config.json") serve_parser.add_argument("--db", default=None, help="Override LadyBugDB path") serve_parser.add_argument("--manifest", default=None, help="Override manifest path") + http_parser = mcp_subparsers.add_parser("http", help="Serve graph tools over Streamable HTTP") + http_parser.add_argument("--repo-root", default=".", help="Repository root containing .codebaseGraph/config.json") + http_parser.add_argument("--config", default=None, help="Path to .codebaseGraph/config.json") + http_parser.add_argument("--db", default=None, help="Override LadyBugDB path") + http_parser.add_argument("--manifest", default=None, help="Override manifest path") + http_parser.add_argument("--host", default="127.0.0.1", help="HTTP bind host; default keeps the server local") + http_parser.add_argument("--port", type=int, default=8765, help="HTTP bind port") + http_parser.add_argument("--path", default="/mcp", help="MCP HTTP endpoint path") args = parser.parse_args(argv) if args.command == "materialize": @@ -59,7 +69,10 @@ def main(argv: Sequence[str] | None = None) -> int: manifest_path=args.manifest, include_fts=not args.no_fts, ) - result = materializer.materialize(mode=args.mode) + try: + result = materializer.materialize(mode=args.mode) + finally: + materializer.close() print(json.dumps(_result_payload(result), indent=2, sort_keys=True)) return 0 if args.command in {"search", "context"}: @@ -80,9 +93,15 @@ def main(argv: Sequence[str] | None = None) -> int: manifest_path=args.manifest, include_fts=True, ) - if not args.no_refresh: - materializer.materialize(mode="changed") - payload = SearchService(materializer.store).search(request) + if args.no_refresh: + with create_ladybug_database(materializer.db_path, include_fts=True, read_only=True) as store: + payload = SearchService(store).search(request) + else: + try: + materializer.materialize(mode="changed") + payload = SearchService(materializer.store).search(request) + finally: + materializer.close() print(json.dumps(payload.as_dict(), indent=2, sort_keys=True)) return 0 if args.command == "setup": @@ -107,6 +126,19 @@ def main(argv: Sequence[str] | None = None) -> int: serve_stdio(repo_root=args.repo_root, config_path=args.config, db_path=args.db, manifest_path=args.manifest) return 0 + if args.command == "mcp" and args.mcp_command == "http": + from codebase_graph.mcp.server import serve_http + + serve_http( + repo_root=args.repo_root, + config_path=args.config, + db_path=args.db, + manifest_path=args.manifest, + host=args.host, + port=args.port, + endpoint_path=args.path, + ) + return 0 parser.error(f"Unknown command: {args.command}") return 2 diff --git a/src/codebase_graph/db/store.py b/src/codebase_graph/db/store.py index 7d88c2d..3b79e47 100644 --- a/src/codebase_graph/db/store.py +++ b/src/codebase_graph/db/store.py @@ -28,9 +28,16 @@ class BulkLoadStats: class LadybugCodeGraphStore: - def __init__(self, db_path: str | Path = ":memory:", *, include_fts: bool = True) -> None: + def __init__( + self, + db_path: str | Path = ":memory:", + *, + include_fts: bool = True, + read_only: bool = False, + ) -> None: self.db_path = db_path self.include_fts = include_fts + self.read_only = read_only try: import real_ladybug as lb except ImportError as exc: @@ -40,9 +47,9 @@ def __init__(self, db_path: str | Path = ":memory:", *, include_fts: bool = True ) from exc self._lb = lb - if str(db_path) != ":memory:": + if str(db_path) != ":memory:" and not read_only: Path(db_path).parent.mkdir(parents=True, exist_ok=True) - self.db = lb.Database(str(db_path)) + self.db = lb.Database(str(db_path), read_only=read_only) self.conn = lb.Connection(self.db) @property @@ -53,6 +60,11 @@ def ensure_schema(self) -> None: for statement in build_ladybug_schema_statements(include_fts=self.include_fts): self._execute_ignoring_existing(statement) + def load_extensions(self) -> None: + for statement in build_ladybug_schema_statements(include_fts=self.include_fts): + if statement.upper().startswith("LOAD "): + self.execute(statement) + def execute(self, statement: str, parameters: dict[str, Any] | None = None) -> Any: if parameters is None: return self.conn.execute(statement) @@ -200,9 +212,17 @@ def _delete_node(self, node_id: str, node_type: str) -> None: ) -def create_ladybug_database(db_path: str | Path = ":memory:", *, include_fts: bool = True) -> LadybugCodeGraphStore: - store = LadybugCodeGraphStore(db_path, include_fts=include_fts) - store.ensure_schema() +def create_ladybug_database( + db_path: str | Path = ":memory:", + *, + include_fts: bool = True, + read_only: bool = False, +) -> LadybugCodeGraphStore: + store = LadybugCodeGraphStore(db_path, include_fts=include_fts, read_only=read_only) + if read_only: + store.load_extensions() + else: + store.ensure_schema() return store diff --git a/src/codebase_graph/ingest/materializer.py b/src/codebase_graph/ingest/materializer.py index 6411a1a..1e56673 100644 --- a/src/codebase_graph/ingest/materializer.py +++ b/src/codebase_graph/ingest/materializer.py @@ -230,6 +230,9 @@ def store(self, value: LadybugCodeGraphStore | None) -> None: self._store = value self._store_injected = value is not None + def close(self) -> None: + self._close_store() + def materialize(self, mode: MaterializeMode = "changed") -> MaterializationResult: if mode not in {"full", "changed"}: raise ValueError(f"Unsupported materialization mode: {mode}") @@ -360,14 +363,17 @@ def _materialize_full_atomic( next_manifest.write(temp_manifest_path) _write_rebuild_marker(marker_path, target_db_path, self.manifest_path) self._close_store() + _unlink_db_sidecars(target_db_path) os.replace(temp_db_path, target_db_path) os.replace(temp_manifest_path, self.manifest_path) + _unlink_db_sidecars(target_db_path) _unlink_if_exists(marker_path) self._store = None except Exception: if temp_store is not None: temp_store.close() _unlink_if_exists(temp_db_path) + _unlink_db_sidecars(temp_db_path) _unlink_if_exists(temp_manifest_path) _unlink_if_exists(temp_manifest_path.with_suffix(temp_manifest_path.suffix + ".tmp")) raise @@ -504,6 +510,11 @@ def _unlink_if_exists(path: Path) -> None: return +def _unlink_db_sidecars(db_path: Path) -> None: + for suffix in (".wal", ".shm"): + _unlink_if_exists(Path(f"{db_path}{suffix}")) + + def _is_excluded(path: Path, source_root: Path) -> bool: parts = path.relative_to(source_root).parts return any(_is_excluded_part(part) for part in parts) diff --git a/src/codebase_graph/mcp/__init__.py b/src/codebase_graph/mcp/__init__.py index fe546b1..ede6b1a 100644 --- a/src/codebase_graph/mcp/__init__.py +++ b/src/codebase_graph/mcp/__init__.py @@ -1,5 +1,5 @@ """MCP server surface for codebaseGraph.""" -from .server import McpGraphServer, handle_tool_call, serve_stdio +from .server import McpGraphServer, handle_tool_call, serve_http, serve_stdio -__all__ = ["McpGraphServer", "handle_tool_call", "serve_stdio"] +__all__ = ["McpGraphServer", "handle_tool_call", "serve_http", "serve_stdio"] diff --git a/src/codebase_graph/mcp/protocol.py b/src/codebase_graph/mcp/protocol.py new file mode 100644 index 0000000..8990a1f --- /dev/null +++ b/src/codebase_graph/mcp/protocol.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from .runtime import GraphRuntimeConfig, package_version +from .tools import UnknownToolError, call_tool_result, tool_specs + +SUPPORTED_PROTOCOL_VERSIONS = ("2025-11-25", "2025-06-18", "2025-03-26", "2024-11-05") +LATEST_PROTOCOL_VERSION = SUPPORTED_PROTOCOL_VERSIONS[0] + + +@dataclass(slots=True) +class ProtocolSession: + protocol_version: str | None = None + initialized: bool = False + + +class McpGraphServer: + def __init__(self, runtime: GraphRuntimeConfig) -> None: + self.runtime = runtime + self.session = ProtocolSession() + + @classmethod + def from_paths( + cls, + *, + repo_root: str = ".", + config_path: str | None = None, + db_path: str | None = None, + manifest_path: str | None = None, + ) -> McpGraphServer: + from .runtime import runtime_config + + runtime = runtime_config( + repo_root=repo_root, + config_path=config_path, + db_path=db_path, + manifest_path=manifest_path, + ) + return cls(runtime) + + def handle_json_rpc(self, message: dict[str, Any]) -> dict[str, Any] | None: + method = str(message.get("method", "")) + request_id = message.get("id") + if method == "notifications/initialized": + self.session.initialized = True + return None + if method.startswith("notifications/"): + return None + try: + if method == "initialize": + result = self._initialize(dict(message.get("params") or {})) + elif method == "ping": + result = {} + elif method == "tools/list": + result = {"tools": tool_specs()} + elif method == "tools/call": + result = self._call_tool(dict(message.get("params") or {})) + else: + return rpc_error(request_id, -32601, f"Unsupported MCP method: {method}") + except UnknownToolError as exc: + return rpc_error(request_id, -32602, str(exc)) + except ValueError as exc: + return rpc_error(request_id, -32602, str(exc)) + except Exception as exc: + return rpc_error(request_id, -32000, str(exc)) + return {"jsonrpc": "2.0", "id": request_id, "result": result} + + def _initialize(self, params: dict[str, Any]) -> dict[str, Any]: + requested = str(params.get("protocolVersion") or "") + protocol_version = negotiate_protocol_version(requested) + self.session.protocol_version = protocol_version + return { + "protocolVersion": protocol_version, + "capabilities": {"tools": {"listChanged": False}}, + "serverInfo": {"name": "codebaseGraph", "version": package_version()}, + } + + def _call_tool(self, params: dict[str, Any]) -> dict[str, Any]: + return call_tool_result( + str(params.get("name", "")), + dict(params.get("arguments") or {}), + runtime=self.runtime, + ) + + +def negotiate_protocol_version(requested: str) -> str: + if requested in SUPPORTED_PROTOCOL_VERSIONS: + return requested + return LATEST_PROTOCOL_VERSION + + +def rpc_error(request_id: Any, code: int, message: str, data: dict[str, Any] | None = None) -> dict[str, Any]: + error: dict[str, Any] = {"code": code, "message": message} + if data is not None: + error["data"] = data + return {"jsonrpc": "2.0", "id": request_id, "error": error} diff --git a/src/codebase_graph/mcp/runtime.py b/src/codebase_graph/mcp/runtime.py new file mode 100644 index 0000000..760f3d2 --- /dev/null +++ b/src/codebase_graph/mcp/runtime.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from dataclasses import dataclass +from importlib.metadata import PackageNotFoundError, version +from pathlib import Path +from typing import Any + +from codebase_graph.db import LadybugCodeGraphStore, create_ladybug_database +from codebase_graph.setup.state import derive_setup_paths, load_setup_config + + +@dataclass(frozen=True, slots=True) +class GraphRuntimeConfig: + repo_root: Path + db_path: Path + manifest_path: Path | None = None + + +def runtime_config( + *, + repo_root: str | Path, + config_path: str | Path | None, + db_path: str | Path | None, + manifest_path: str | Path | None, +) -> GraphRuntimeConfig: + root = Path(repo_root).expanduser().resolve() + config = Path(config_path).expanduser().resolve() if config_path else derive_setup_paths(root).config_path + payload: dict[str, Any] = {} + if config.exists(): + payload = load_setup_config(config) + elif db_path is None: + raise FileNotFoundError(f"codebaseGraph setup config is missing: {config}") + resolved_db = Path(db_path or payload["database_path"]).expanduser().resolve() + resolved_manifest = ( + Path(manifest_path or payload.get("manifest_path", "")).expanduser().resolve() + if (manifest_path or payload.get("manifest_path")) + else None + ) + if not resolved_db.exists(): + raise FileNotFoundError(f"codebaseGraph database is missing: {resolved_db}") + return GraphRuntimeConfig(repo_root=root, db_path=resolved_db, manifest_path=resolved_manifest) + + +def open_graph_store(runtime: GraphRuntimeConfig) -> LadybugCodeGraphStore: + return create_ladybug_database(runtime.db_path, include_fts=True, read_only=True) + + +def package_version() -> str: + try: + return version("codebase-graph") + except PackageNotFoundError: + return "0.1.0" diff --git a/src/codebase_graph/mcp/server.py b/src/codebase_graph/mcp/server.py index eccc1dd..e1ffff5 100644 --- a/src/codebase_graph/mcp/server.py +++ b/src/codebase_graph/mcp/server.py @@ -1,124 +1,10 @@ from __future__ import annotations -import json -import re -import sys -from dataclasses import dataclass -from importlib.metadata import PackageNotFoundError, version -from pathlib import Path -from typing import Any - -from codebase_graph.db import LadybugCodeGraphStore, create_ladybug_database -from codebase_graph.ontology import QUERY_HELPERS, schema_payload -from codebase_graph.reasoning import CompactContextBuilder -from codebase_graph.retrieval import SearchRequest, SearchService -from codebase_graph.setup.state import derive_setup_paths, load_setup_config - -READ_ONLY_DENY_RE = re.compile( - r"\b(CREATE|DELETE|SET|MERGE|DROP|COPY|INSERT|LOAD|INSTALL|DETACH|REMOVE|ALTER|RENAME)\b", - re.IGNORECASE, -) - - -@dataclass(frozen=True, slots=True) -class GraphRuntimeConfig: - repo_root: Path - db_path: Path - manifest_path: Path | None = None - - -class McpGraphServer: - def __init__(self, runtime: GraphRuntimeConfig) -> None: - self.runtime = runtime - - @classmethod - def from_paths( - cls, - *, - repo_root: str | Path = ".", - config_path: str | Path | None = None, - db_path: str | Path | None = None, - manifest_path: str | Path | None = None, - ) -> McpGraphServer: - runtime = _runtime_config( - repo_root=repo_root, - config_path=config_path, - db_path=db_path, - manifest_path=manifest_path, - ) - return cls(runtime) - - def handle_json_rpc(self, message: dict[str, Any]) -> dict[str, Any] | None: - method = str(message.get("method", "")) - request_id = message.get("id") - if method.startswith("notifications/"): - return None - try: - if method == "initialize": - result = { - "protocolVersion": "2024-11-05", - "capabilities": {"tools": {}}, - "serverInfo": {"name": "codebaseGraph", "version": _package_version()}, - } - elif method == "ping": - result = {} - elif method == "tools/list": - result = {"tools": _tool_specs()} - elif method == "tools/call": - params = dict(message.get("params") or {}) - payload = handle_tool_call( - str(params.get("name", "")), - dict(params.get("arguments") or {}), - runtime=self.runtime, - ) - result = _tool_result(payload) - else: - return _rpc_error(request_id, -32601, f"Unsupported MCP method: {method}") - except Exception as exc: - return _rpc_error(request_id, -32000, str(exc)) - return {"jsonrpc": "2.0", "id": request_id, "result": result} - - -def handle_tool_call(name: str, arguments: dict[str, Any], *, runtime: GraphRuntimeConfig) -> dict[str, Any]: - if name == "graph_health": - return _health(runtime) - if name == "graph_schema": - return schema_payload() - if name == "graph_query_helpers": - return {"query_helpers": [helper.as_dict() for helper in QUERY_HELPERS]} - if name == "graph_search": - with _store(runtime) as store: - request = _search_request(arguments) - return SearchService(store).search(request).as_dict() - if name == "graph_context": - with _store(runtime) as store: - return _context_payload(store, arguments) - if name == "graph_query": - with _store(runtime) as store: - return _query_payload(store, arguments) - raise ValueError(f"Unknown codebaseGraph MCP tool: {name}") - - -def serve_stdio( - *, - repo_root: str | Path = ".", - config_path: str | Path | None = None, - db_path: str | Path | None = None, - manifest_path: str | Path | None = None, -) -> None: - server = McpGraphServer.from_paths( - repo_root=repo_root, - config_path=config_path, - db_path=db_path, - manifest_path=manifest_path, - ) - while True: - message = _read_message(sys.stdin.buffer) - if message is None: - return - response = server.handle_json_rpc(message) - if response is not None: - _write_message(sys.stdout.buffer, response) +from .protocol import LATEST_PROTOCOL_VERSION, SUPPORTED_PROTOCOL_VERSIONS, McpGraphServer, negotiate_protocol_version +from .runtime import GraphRuntimeConfig +from .tools import handle_tool_call +from .transports.http import build_http_server, serve_http +from .transports.stdio import serve_stdio def main() -> int: @@ -134,221 +20,14 @@ def main() -> int: return 0 -def _runtime_config( - *, - repo_root: str | Path, - config_path: str | Path | None, - db_path: str | Path | None, - manifest_path: str | Path | None, -) -> GraphRuntimeConfig: - root = Path(repo_root).expanduser().resolve() - config = Path(config_path).expanduser().resolve() if config_path else derive_setup_paths(root).config_path - payload: dict[str, Any] = {} - if config.exists(): - payload = load_setup_config(config) - elif db_path is None: - raise FileNotFoundError(f"codebaseGraph setup config is missing: {config}") - resolved_db = Path(db_path or payload["database_path"]).expanduser().resolve() - resolved_manifest = Path(manifest_path or payload.get("manifest_path", "")).expanduser().resolve() if (manifest_path or payload.get("manifest_path")) else None - if not resolved_db.exists(): - raise FileNotFoundError(f"codebaseGraph database is missing: {resolved_db}") - return GraphRuntimeConfig(repo_root=root, db_path=resolved_db, manifest_path=resolved_manifest) - - -def _store(runtime: GraphRuntimeConfig) -> LadybugCodeGraphStore: - return create_ladybug_database(runtime.db_path, include_fts=True) - - -def _health(runtime: GraphRuntimeConfig) -> dict[str, Any]: - return { - "ok": runtime.db_path.exists(), - "repo_root": runtime.repo_root.as_posix(), - "database_path": runtime.db_path.as_posix(), - "manifest_path": runtime.manifest_path.as_posix() if runtime.manifest_path else None, - } - - -def _search_request(arguments: dict[str, Any]) -> SearchRequest: - request = SearchRequest( - query=str(arguments.get("query", "")), - limit=int(arguments.get("limit", 3)), - profile=str(arguments.get("profile", "brief")), - budget=int(arguments.get("budget", 600)), - max_depth=_optional_int(arguments.get("max_depth")), - ) - request.validate() - return request - - -def _context_payload(store: LadybugCodeGraphStore, arguments: dict[str, Any]) -> dict[str, Any]: - node_id = str(arguments.get("node_id") or "") - node_type = str(arguments.get("node_type") or "") - if node_id and node_type: - profile = str(arguments.get("profile", "brief")) - context = CompactContextBuilder(store).build( - node_id, - node_type, - profile=profile, - limit=int(arguments.get("limit", 3)), - budget=int(arguments.get("budget", 600)), - max_depth=_optional_int(arguments.get("max_depth")), - ) - return { - "node_id": node_id, - "node_type": node_type, - "profile": profile, - "context": [node.as_dict() for node in context], - } - return SearchService(store).search(_search_request(arguments)).as_dict() - - -def _query_payload(store: LadybugCodeGraphStore, arguments: dict[str, Any]) -> dict[str, Any]: - statement = str(arguments.get("statement") or arguments.get("query") or "").strip() - if not statement: - raise ValueError("graph_query requires a non-empty statement") - _validate_read_only_statement(statement) - parameters = arguments.get("parameters") or {} - if not isinstance(parameters, dict): - raise ValueError("graph_query parameters must be a JSON object") - limit = int(arguments.get("limit", 100)) - rows = store.execute(statement, parameters).get_all() - return { - "statement": statement, - "row_count": len(rows), - "rows": [_row_values(row) for row in rows[:limit]], - "truncated": len(rows) > limit, - } - - -def _validate_read_only_statement(statement: str) -> None: - stripped = statement.strip().rstrip(";") - if ";" in stripped: - raise ValueError("graph_query accepts one read-only statement at a time") - match = READ_ONLY_DENY_RE.search(stripped) - if match is not None: - raise ValueError(f"graph_query is read-only; blocked keyword: {match.group(1).upper()}") - - -def _row_values(row: Any) -> list[Any]: - try: - return [_json_safe(value) for value in row] - except TypeError: - return [_json_safe(row)] - - -def _json_safe(value: Any) -> Any: - if value is None or isinstance(value, (str, int, float, bool)): - return value - if isinstance(value, (list, tuple)): - return [_json_safe(item) for item in value] - if isinstance(value, dict): - return {str(key): _json_safe(item) for key, item in value.items()} - return str(value) - - -def _tool_result(payload: dict[str, Any]) -> dict[str, Any]: - return { - "content": [{"type": "text", "text": json.dumps(payload, indent=2, sort_keys=True)}], - "structuredContent": payload, - "isError": False, - } - - -def _tool_specs() -> list[dict[str, Any]]: - return [ - { - "name": "graph_health", - "description": "Check the configured codebaseGraph database path and manifest path.", - "inputSchema": {"type": "object", "properties": {}, "additionalProperties": False}, - }, - { - "name": "graph_search", - "description": "Search code, documentation, paths, and dependencies with compact graph context.", - "inputSchema": _search_schema(required=("query",)), - }, - { - "name": "graph_context", - "description": "Return compact context for a search query or explicit node_id/node_type pair.", - "inputSchema": _search_schema(required=()), - }, - { - "name": "graph_schema", - "description": "Return ontology schema, search indexes, context profiles, and query helper metadata.", - "inputSchema": {"type": "object", "properties": {}, "additionalProperties": False}, - }, - { - "name": "graph_query_helpers", - "description": "Return named read-only query helpers for common graph exploration tasks.", - "inputSchema": {"type": "object", "properties": {}, "additionalProperties": False}, - }, - { - "name": "graph_query", - "description": "Execute a restricted read-only graph query against the configured database.", - "inputSchema": { - "type": "object", - "properties": { - "statement": {"type": "string"}, - "parameters": {"type": "object"}, - "limit": {"type": "integer", "minimum": 1}, - }, - "required": ["statement"], - "additionalProperties": False, - }, - }, - ] - - -def _search_schema(*, required: tuple[str, ...]) -> dict[str, Any]: - return { - "type": "object", - "properties": { - "query": {"type": "string"}, - "limit": {"type": "integer", "minimum": 1}, - "profile": {"type": "string"}, - "budget": {"type": "integer", "minimum": 0}, - "max_depth": {"type": "integer", "minimum": 0}, - "node_id": {"type": "string"}, - "node_type": {"type": "string"}, - }, - "required": list(required), - "additionalProperties": False, - } - - -def _optional_int(value: Any) -> int | None: - if value is None or value == "": - return None - return int(value) - - -def _rpc_error(request_id: Any, code: int, message: str) -> dict[str, Any]: - return {"jsonrpc": "2.0", "id": request_id, "error": {"code": code, "message": message}} - - -def _read_message(stream: Any) -> dict[str, Any] | None: - line = stream.readline() - if not line: - return None - if line.lower().startswith(b"content-length:"): - length = int(line.split(b":", 1)[1].strip()) - while True: - header = stream.readline() - if header in {b"\r\n", b"\n", b""}: - break - body = stream.read(length) - return json.loads(body.decode("utf-8")) - return json.loads(line.decode("utf-8")) - - -def _write_message(stream: Any, message: dict[str, Any]) -> None: - body = json.dumps(message, separators=(",", ":"), sort_keys=True).encode("utf-8") - stream.write(f"Content-Length: {len(body)}\r\n\r\n".encode("ascii")) - stream.write(body) - stream.flush() - - -def _package_version() -> str: - try: - return version("codebase-graph") - except PackageNotFoundError: - return "0.1.0" +__all__ = [ + "LATEST_PROTOCOL_VERSION", + "SUPPORTED_PROTOCOL_VERSIONS", + "GraphRuntimeConfig", + "McpGraphServer", + "build_http_server", + "handle_tool_call", + "negotiate_protocol_version", + "serve_http", + "serve_stdio", +] diff --git a/src/codebase_graph/mcp/tools.py b/src/codebase_graph/mcp/tools.py new file mode 100644 index 0000000..f6bd39e --- /dev/null +++ b/src/codebase_graph/mcp/tools.py @@ -0,0 +1,228 @@ +from __future__ import annotations + +import json +import re +from typing import Any + +from codebase_graph.db import LadybugCodeGraphStore +from codebase_graph.ontology import QUERY_HELPERS, schema_payload +from codebase_graph.reasoning import CompactContextBuilder +from codebase_graph.retrieval import SearchRequest, SearchService + +from .runtime import GraphRuntimeConfig, open_graph_store + +READ_ONLY_DENY_RE = re.compile( + r"\b(CREATE|DELETE|SET|MERGE|DROP|COPY|INSERT|LOAD|INSTALL|DETACH|REMOVE|ALTER|RENAME)\b", + re.IGNORECASE, +) + + +class UnknownToolError(ValueError): + pass + + +def handle_tool_call(name: str, arguments: dict[str, Any], *, runtime: GraphRuntimeConfig) -> dict[str, Any]: + if name == "graph_health": + return _health(runtime) + if name == "graph_schema": + return schema_payload() + if name == "graph_query_helpers": + return {"query_helpers": [helper.as_dict() for helper in QUERY_HELPERS]} + if name == "graph_search": + with open_graph_store(runtime) as store: + request = _search_request(arguments) + return SearchService(store).search(request).as_dict() + if name == "graph_context": + with open_graph_store(runtime) as store: + return _context_payload(store, arguments) + if name == "graph_query": + with open_graph_store(runtime) as store: + return _query_payload(store, arguments) + raise UnknownToolError(f"Unknown codebaseGraph MCP tool: {name}") + + +def call_tool_result(name: str, arguments: dict[str, Any], *, runtime: GraphRuntimeConfig) -> dict[str, Any]: + try: + payload = handle_tool_call(name, arguments, runtime=runtime) + except UnknownToolError: + raise + except Exception as exc: + return tool_error_result(name, exc) + return tool_result(payload) + + +def tool_result(payload: dict[str, Any]) -> dict[str, Any]: + return { + "content": [{"type": "text", "text": json.dumps(payload, indent=2, sort_keys=True)}], + "structuredContent": payload, + "isError": False, + } + + +def tool_error_result(name: str, exc: Exception) -> dict[str, Any]: + payload = { + "error": { + "tool": name, + "type": exc.__class__.__name__, + "message": str(exc), + } + } + return { + "content": [{"type": "text", "text": f"{name} failed: {exc}"}], + "structuredContent": payload, + "isError": True, + } + + +def tool_specs() -> list[dict[str, Any]]: + return [ + { + "name": "graph_health", + "description": "Check the configured codebaseGraph database path and manifest path.", + "inputSchema": {"type": "object", "properties": {}, "additionalProperties": False}, + }, + { + "name": "graph_search", + "description": "Search code, documentation, paths, and dependencies with compact graph context.", + "inputSchema": _search_schema(required=("query",)), + }, + { + "name": "graph_context", + "description": "Return compact context for a search query or explicit node_id/node_type pair.", + "inputSchema": _search_schema(required=()), + }, + { + "name": "graph_schema", + "description": "Return ontology schema, search indexes, context profiles, and query helper metadata.", + "inputSchema": {"type": "object", "properties": {}, "additionalProperties": False}, + }, + { + "name": "graph_query_helpers", + "description": "Return named read-only query helpers for common graph exploration tasks.", + "inputSchema": {"type": "object", "properties": {}, "additionalProperties": False}, + }, + { + "name": "graph_query", + "description": "Execute a restricted read-only graph query against the configured database.", + "inputSchema": { + "type": "object", + "properties": { + "statement": {"type": "string"}, + "parameters": {"type": "object"}, + "limit": {"type": "integer", "minimum": 1}, + }, + "required": ["statement"], + "additionalProperties": False, + }, + }, + ] + + +def _health(runtime: GraphRuntimeConfig) -> dict[str, Any]: + return { + "ok": runtime.db_path.exists(), + "repo_root": runtime.repo_root.as_posix(), + "database_path": runtime.db_path.as_posix(), + "manifest_path": runtime.manifest_path.as_posix() if runtime.manifest_path else None, + } + + +def _search_request(arguments: dict[str, Any]) -> SearchRequest: + request = SearchRequest( + query=str(arguments.get("query", "")), + limit=int(arguments.get("limit", 3)), + profile=str(arguments.get("profile", "brief")), + budget=int(arguments.get("budget", 600)), + max_depth=_optional_int(arguments.get("max_depth")), + ) + request.validate() + return request + + +def _context_payload(store: LadybugCodeGraphStore, arguments: dict[str, Any]) -> dict[str, Any]: + node_id = str(arguments.get("node_id") or "") + node_type = str(arguments.get("node_type") or "") + if node_id and node_type: + profile = str(arguments.get("profile", "brief")) + context = CompactContextBuilder(store).build( + node_id, + node_type, + profile=profile, + limit=int(arguments.get("limit", 3)), + budget=int(arguments.get("budget", 600)), + max_depth=_optional_int(arguments.get("max_depth")), + ) + return { + "node_id": node_id, + "node_type": node_type, + "profile": profile, + "context": [node.as_dict() for node in context], + } + return SearchService(store).search(_search_request(arguments)).as_dict() + + +def _query_payload(store: LadybugCodeGraphStore, arguments: dict[str, Any]) -> dict[str, Any]: + statement = str(arguments.get("statement") or arguments.get("query") or "").strip() + if not statement: + raise ValueError("graph_query requires a non-empty statement") + _validate_read_only_statement(statement) + parameters = arguments.get("parameters") or {} + if not isinstance(parameters, dict): + raise ValueError("graph_query parameters must be a JSON object") + limit = int(arguments.get("limit", 100)) + rows = store.execute(statement, parameters).get_all() + return { + "statement": statement, + "row_count": len(rows), + "rows": [_row_values(row) for row in rows[:limit]], + "truncated": len(rows) > limit, + } + + +def _validate_read_only_statement(statement: str) -> None: + stripped = statement.strip().rstrip(";") + if ";" in stripped: + raise ValueError("graph_query accepts one read-only statement at a time") + match = READ_ONLY_DENY_RE.search(stripped) + if match is not None: + raise ValueError(f"graph_query is read-only; blocked keyword: {match.group(1).upper()}") + + +def _row_values(row: Any) -> list[Any]: + try: + return [_json_safe(value) for value in row] + except TypeError: + return [_json_safe(row)] + + +def _json_safe(value: Any) -> Any: + if value is None or isinstance(value, (str, int, float, bool)): + return value + if isinstance(value, (list, tuple)): + return [_json_safe(item) for item in value] + if isinstance(value, dict): + return {str(key): _json_safe(item) for key, item in value.items()} + return str(value) + + +def _search_schema(*, required: tuple[str, ...]) -> dict[str, Any]: + return { + "type": "object", + "properties": { + "query": {"type": "string"}, + "limit": {"type": "integer", "minimum": 1}, + "profile": {"type": "string"}, + "budget": {"type": "integer", "minimum": 0}, + "max_depth": {"type": "integer", "minimum": 0}, + "node_id": {"type": "string"}, + "node_type": {"type": "string"}, + }, + "required": list(required), + "additionalProperties": False, + } + + +def _optional_int(value: Any) -> int | None: + if value is None or value == "": + return None + return int(value) diff --git a/src/codebase_graph/mcp/transports/__init__.py b/src/codebase_graph/mcp/transports/__init__.py new file mode 100644 index 0000000..8d63218 --- /dev/null +++ b/src/codebase_graph/mcp/transports/__init__.py @@ -0,0 +1 @@ +"""Transport implementations for the codebaseGraph MCP server.""" diff --git a/src/codebase_graph/mcp/transports/http.py b/src/codebase_graph/mcp/transports/http.py new file mode 100644 index 0000000..7e74bad --- /dev/null +++ b/src/codebase_graph/mcp/transports/http.py @@ -0,0 +1,142 @@ +from __future__ import annotations + +import json +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any +from urllib.parse import urlparse + +from codebase_graph.mcp.protocol import SUPPORTED_PROTOCOL_VERSIONS, McpGraphServer, rpc_error + +LOCAL_ORIGINS = {"localhost", "127.0.0.1", "::1"} + + +class McpHttpServer(ThreadingHTTPServer): + def __init__(self, server_address: tuple[str, int], handler: type[BaseHTTPRequestHandler]) -> None: + super().__init__(server_address, handler) + self.mcp_server: McpGraphServer + self.endpoint_path: str + + +def build_http_server( + *, + repo_root: str | Path = ".", + config_path: str | Path | None = None, + db_path: str | Path | None = None, + manifest_path: str | Path | None = None, + host: str = "127.0.0.1", + port: int = 8765, + endpoint_path: str = "/mcp", +) -> McpHttpServer: + graph_server = McpGraphServer.from_paths( + repo_root=repo_root, + config_path=config_path, + db_path=db_path, + manifest_path=manifest_path, + ) + httpd = McpHttpServer((host, port), _McpHttpHandler) + httpd.mcp_server = graph_server + httpd.endpoint_path = endpoint_path + return httpd + + +def serve_http( + *, + repo_root: str | Path = ".", + config_path: str | Path | None = None, + db_path: str | Path | None = None, + manifest_path: str | Path | None = None, + host: str = "127.0.0.1", + port: int = 8765, + endpoint_path: str = "/mcp", +) -> None: + server = build_http_server( + repo_root=repo_root, + config_path=config_path, + db_path=db_path, + manifest_path=manifest_path, + host=host, + port=port, + endpoint_path=endpoint_path, + ) + try: + server.serve_forever() + finally: + server.server_close() + + +class _McpHttpHandler(BaseHTTPRequestHandler): + server: McpHttpServer + + def do_POST(self) -> None: + if not self._request_path_matches() or not self._valid_origin(): + return + if not self._valid_protocol_header(): + return + try: + length = int(self.headers.get("Content-Length", "0")) + message = json.loads(self.rfile.read(length).decode("utf-8")) + except Exception as exc: + self._send_json(rpc_error(None, -32700, f"Invalid JSON-RPC payload: {exc}"), status=HTTPStatus.BAD_REQUEST) + return + if not isinstance(message, dict): + self._send_json(rpc_error(None, -32600, "JSON-RPC payload must be an object"), status=HTTPStatus.BAD_REQUEST) + return + response = self.server.mcp_server.handle_json_rpc(message) + if response is None: + self.send_response(HTTPStatus.ACCEPTED) + self.end_headers() + return + self._send_json(response) + + def do_GET(self) -> None: + if not self._request_path_matches() or not self._valid_origin(): + return + self.send_response(HTTPStatus.METHOD_NOT_ALLOWED) + self.send_header("Allow", "POST") + self.end_headers() + + def log_message(self, format: str, *args: Any) -> None: + return + + def _request_path_matches(self) -> bool: + if urlparse(self.path).path == self.server.endpoint_path: + return True + self._send_json(rpc_error(None, -32601, "MCP endpoint not found"), status=HTTPStatus.NOT_FOUND) + return False + + def _valid_origin(self) -> bool: + origin = self.headers.get("Origin") + if not origin: + return True + hostname = urlparse(origin).hostname + if hostname in LOCAL_ORIGINS: + return True + self._send_json(rpc_error(None, -32000, "Forbidden origin"), status=HTTPStatus.FORBIDDEN) + return False + + def _valid_protocol_header(self) -> bool: + requested = self.headers.get("MCP-Protocol-Version") + if requested is None: + return True + if requested in SUPPORTED_PROTOCOL_VERSIONS: + return True + self._send_json( + rpc_error( + None, + -32602, + "Unsupported MCP protocol version", + {"supported": list(SUPPORTED_PROTOCOL_VERSIONS), "requested": requested}, + ), + status=HTTPStatus.BAD_REQUEST, + ) + return False + + def _send_json(self, payload: dict[str, Any], *, status: HTTPStatus = HTTPStatus.OK) -> None: + body = json.dumps(payload, separators=(",", ":"), sort_keys=True).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) diff --git a/src/codebase_graph/mcp/transports/stdio.py b/src/codebase_graph/mcp/transports/stdio.py new file mode 100644 index 0000000..23e480b --- /dev/null +++ b/src/codebase_graph/mcp/transports/stdio.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import Any, BinaryIO + +from codebase_graph.mcp.protocol import McpGraphServer + + +def serve_stdio( + *, + repo_root: str | Path = ".", + config_path: str | Path | None = None, + db_path: str | Path | None = None, + manifest_path: str | Path | None = None, +) -> None: + server = McpGraphServer.from_paths( + repo_root=repo_root, + config_path=config_path, + db_path=db_path, + manifest_path=manifest_path, + ) + while True: + message = read_message(sys.stdin.buffer) + if message is None: + return + response = server.handle_json_rpc(message) + if response is not None: + write_message(sys.stdout.buffer, response) + + +def read_message(stream: BinaryIO) -> dict[str, Any] | None: + line = stream.readline() + if not line: + return None + if line.lower().startswith(b"content-length:"): + length = int(line.split(b":", 1)[1].strip()) + while True: + header = stream.readline() + if header in {b"\r\n", b"\n", b""}: + break + body = stream.read(length) + return json.loads(body.decode("utf-8")) + return json.loads(line.decode("utf-8")) + + +def write_message(stream: BinaryIO, message: dict[str, Any]) -> None: + body = json.dumps(message, separators=(",", ":"), sort_keys=True).encode("utf-8") + stream.write(f"Content-Length: {len(body)}\r\n\r\n".encode("ascii")) + stream.write(body) + stream.flush() diff --git a/src/codebase_graph/setup/clients/__init__.py b/src/codebase_graph/setup/clients/__init__.py new file mode 100644 index 0000000..a482497 --- /dev/null +++ b/src/codebase_graph/setup/clients/__init__.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +from .base import ClientConfigAdapter, RenderedClientConfig +from .codex import CodexAdapter +from .hermes import HermesAdapter +from .json_clients import ClaudeAdapter, ClaudeProjectAdapter, GenericAdapter, LmStudioAdapter, OpenClawAdapter + +ADAPTERS: dict[str, ClientConfigAdapter] = { + adapter.client_id: adapter + for adapter in ( + CodexAdapter(), + ClaudeAdapter(), + ClaudeProjectAdapter(), + LmStudioAdapter(), + HermesAdapter(), + OpenClawAdapter(), + GenericAdapter(), + ) +} + + +def get_client_adapter(client_id: str) -> ClientConfigAdapter: + try: + return ADAPTERS[client_id] + except KeyError as exc: + supported = ", ".join(sorted([*ADAPTERS, "none"])) + raise ValueError(f"Unsupported MCP client: {client_id}. Supported clients: {supported}") from exc + + +def supported_client_ids() -> tuple[str, ...]: + return tuple(sorted([*ADAPTERS, "none"])) + + +__all__ = ["ADAPTERS", "ClientConfigAdapter", "RenderedClientConfig", "get_client_adapter", "supported_client_ids"] diff --git a/src/codebase_graph/setup/clients/base.py b/src/codebase_graph/setup/clients/base.py new file mode 100644 index 0000000..53db3e3 --- /dev/null +++ b/src/codebase_graph/setup/clients/base.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Protocol + +from codebase_graph.setup.descriptor import McpServerDescriptor + + +@dataclass(frozen=True, slots=True) +class RenderedClientConfig: + text: str + action: str + entry: dict[str, Any] + patch: Any + payload: Any + + +class ClientConfigAdapter(Protocol): + client_id: str + + def default_config_path(self, descriptor: McpServerDescriptor) -> Path: + ... + + def render(self, existing_text: str | None, descriptor: McpServerDescriptor) -> RenderedClientConfig: + ... + + +def action_for_server(previous: Any, next_value: Any, *, file_exists: bool) -> str: + if previous is None: + return "created" + if previous == next_value: + return "unchanged" + return "updated" diff --git a/src/codebase_graph/setup/clients/codex.py b/src/codebase_graph/setup/clients/codex.py new file mode 100644 index 0000000..abdde9a --- /dev/null +++ b/src/codebase_graph/setup/clients/codex.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import os +import re +from pathlib import Path +from typing import Any + +from codebase_graph.setup.descriptor import McpServerDescriptor + +from .base import RenderedClientConfig + + +class CodexAdapter: + client_id = "codex" + + def default_config_path(self, descriptor: McpServerDescriptor) -> Path: + base = Path(os.environ.get("CODEX_HOME", Path.home() / ".codex")) + return base / "config.toml" + + def render(self, existing_text: str | None, descriptor: McpServerDescriptor) -> RenderedClientConfig: + entry = descriptor.stdio_entry(include_timeout=True) + patch = _toml_block(descriptor, entry) + next_text, previous = _upsert_toml_block(existing_text or "", descriptor.name, patch) + if previous is None: + action = "created" + elif previous == patch.rstrip(): + action = "unchanged" + else: + action = "updated" + if existing_text == next_text: + action = "unchanged" + return RenderedClientConfig(text=next_text, action=action, entry=entry, patch=patch, payload=patch) + + +def _upsert_toml_block(existing: str, server_name: str, block: str) -> tuple[str, str | None]: + lines = existing.splitlines() + start: int | None = None + end = len(lines) + header_re = re.compile(rf"^\[mcp_servers\.{re.escape(server_name)}(?:\.env)?\]\s*$") + any_header_re = re.compile(r"^\[[^\]]+\]\s*$") + for index, line in enumerate(lines): + if header_re.match(line): + start = index + break + if start is None: + prefix = existing.rstrip() + separator = "\n\n" if prefix else "" + return f"{prefix}{separator}{block}", None + for index in range(start + 1, len(lines)): + if any_header_re.match(lines[index]) and not header_re.match(lines[index]): + end = index + break + previous = "\n".join(lines[start:end]).rstrip() + next_lines = [*lines[:start], *block.rstrip().splitlines(), *lines[end:]] + return "\n".join(next_lines).rstrip() + "\n", previous + + +def _toml_block(descriptor: McpServerDescriptor, entry: dict[str, Any]) -> str: + lines = [ + f"[mcp_servers.{descriptor.name}]", + f"command = {_toml_string(entry['command'])}", + f"args = {_toml_array(entry['args'])}", + f"startup_timeout_sec = {int(entry['startup_timeout_sec'])}", + ] + if descriptor.cwd: + lines.append(f"cwd = {_toml_string(descriptor.cwd)}") + if descriptor.env: + lines.append("") + lines.append(f"[mcp_servers.{descriptor.name}.env]") + for key, value in sorted(descriptor.env.items()): + lines.append(f"{key} = {_toml_string(value)}") + return "\n".join(lines) + "\n" + + +def _toml_array(values: list[str]) -> str: + return "[" + ", ".join(_toml_string(value) for value in values) + "]" + + +def _toml_string(value: str) -> str: + escaped = value.replace("\\", "\\\\").replace('"', '\\"') + return f'"{escaped}"' diff --git a/src/codebase_graph/setup/clients/hermes.py b/src/codebase_graph/setup/clients/hermes.py new file mode 100644 index 0000000..018621b --- /dev/null +++ b/src/codebase_graph/setup/clients/hermes.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from codebase_graph.setup.descriptor import McpServerDescriptor + +from .base import RenderedClientConfig + +START_MARKER = "# codebaseGraph MCP server start" +END_MARKER = "# codebaseGraph MCP server end" + + +class HermesAdapter: + client_id = "hermes" + + def default_config_path(self, descriptor: McpServerDescriptor) -> Path: + return Path.home() / ".config" / "hermes" / "mcp_servers.yaml" + + def render(self, existing_text: str | None, descriptor: McpServerDescriptor) -> RenderedClientConfig: + entry = descriptor.stdio_entry(include_type=True) + patch = _yaml_block(descriptor, entry) + next_text, previous = _upsert_marked_block(existing_text or "", patch) + if previous is None: + action = "created" + elif previous == patch.rstrip(): + action = "unchanged" + else: + action = "updated" + if existing_text == next_text: + action = "unchanged" + return RenderedClientConfig(text=next_text, action=action, entry=entry, patch=patch, payload=patch) + + +def _upsert_marked_block(existing: str, block: str) -> tuple[str, str | None]: + start = existing.find(START_MARKER) + end = existing.find(END_MARKER) + if start == -1 or end == -1 or end < start: + prefix = existing.rstrip() + separator = "\n\n" if prefix else "" + return f"{prefix}{separator}{block}", None + after_end = end + len(END_MARKER) + previous = existing[start:after_end].rstrip() + next_text = existing[:start].rstrip() + "\n\n" + block.rstrip() + "\n\n" + existing[after_end:].lstrip() + return next_text.rstrip() + "\n", previous + + +def _yaml_block(descriptor: McpServerDescriptor, entry: dict[str, Any]) -> str: + lines = [ + START_MARKER, + "mcp_servers:", + f" {descriptor.name}:", + " type: stdio", + f" command: {_yaml_scalar(entry['command'])}", + " args:", + ] + for arg in entry["args"]: + lines.append(f" - {_yaml_scalar(arg)}") + if descriptor.cwd: + lines.append(f" cwd: {_yaml_scalar(descriptor.cwd)}") + if descriptor.env: + lines.append(" env:") + for key, value in sorted(descriptor.env.items()): + lines.append(f" {key}: {_yaml_scalar(value)}") + lines.append(END_MARKER) + return "\n".join(lines) + "\n" + + +def _yaml_scalar(value: str) -> str: + escaped = value.replace("\\", "\\\\").replace('"', '\\"') + return f'"{escaped}"' diff --git a/src/codebase_graph/setup/clients/json_clients.py b/src/codebase_graph/setup/clients/json_clients.py new file mode 100644 index 0000000..7886be1 --- /dev/null +++ b/src/codebase_graph/setup/clients/json_clients.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import json +import os +from copy import deepcopy +from pathlib import Path +from typing import Any + +from codebase_graph.setup.descriptor import McpServerDescriptor + +from .base import RenderedClientConfig, action_for_server + + +class JsonMcpServersAdapter: + client_id = "generic" + include_type = True + root_path = ("mcpServers",) + + def default_config_path(self, descriptor: McpServerDescriptor) -> Path: + return Path.home() / ".config" / "mcp" / "mcp.json" + + def entry(self, descriptor: McpServerDescriptor) -> dict[str, Any]: + return descriptor.stdio_entry(include_type=self.include_type) + + def render(self, existing_text: str | None, descriptor: McpServerDescriptor) -> RenderedClientConfig: + payload = _read_json_text(existing_text) + next_payload = deepcopy(payload) + container = _container(next_payload, self.root_path) + entry = self.entry(descriptor) + previous = container.get(descriptor.name) + container[descriptor.name] = entry + action = action_for_server(previous, entry, file_exists=existing_text is not None) + text = json.dumps(next_payload, indent=2, sort_keys=True) + "\n" + if existing_text == text: + action = "unchanged" + return RenderedClientConfig(text=text, action=action, entry=entry, patch=next_payload, payload=next_payload) + + +class ClaudeAdapter(JsonMcpServersAdapter): + client_id = "claude" + include_type = False + + def default_config_path(self, descriptor: McpServerDescriptor) -> Path: + mac_path = Path.home() / "Library" / "Application Support" / "Claude" / "claude_desktop_config.json" + if mac_path.parent.exists(): + return mac_path + return Path.home() / ".config" / "claude" / "claude_desktop_config.json" + + +class ClaudeProjectAdapter(JsonMcpServersAdapter): + client_id = "claude-project" + + def default_config_path(self, descriptor: McpServerDescriptor) -> Path: + if descriptor.repo_root: + return Path(descriptor.repo_root) / ".mcp.json" + return Path.cwd() / ".mcp.json" + + +class LmStudioAdapter(JsonMcpServersAdapter): + client_id = "lmstudio" + + def default_config_path(self, descriptor: McpServerDescriptor) -> Path: + return Path.home() / ".lmstudio" / "mcp.json" + + +class GenericAdapter(JsonMcpServersAdapter): + client_id = "generic" + include_type = False + + +class OpenClawAdapter(JsonMcpServersAdapter): + client_id = "openclaw" + root_path = ("mcp", "servers") + + def default_config_path(self, descriptor: McpServerDescriptor) -> Path: + return Path(os.environ.get("OPENCLAW_HOME", Path.home() / ".openclaw")) / "mcp.json5" + + +def _read_json_text(existing_text: str | None) -> dict[str, Any]: + if existing_text is None or not existing_text.strip(): + return {} + payload = json.loads(existing_text) + if not isinstance(payload, dict): + raise ValueError("MCP config must contain a JSON object") + return payload + + +def _container(payload: dict[str, Any], path: tuple[str, ...]) -> dict[str, Any]: + cursor = payload + for key in path: + next_value = cursor.setdefault(key, {}) + if not isinstance(next_value, dict): + raise ValueError(f"MCP config key must contain an object: {'.'.join(path)}") + cursor = next_value + return cursor diff --git a/src/codebase_graph/setup/descriptor.py b/src/codebase_graph/setup/descriptor.py new file mode 100644 index 0000000..781ac76 --- /dev/null +++ b/src/codebase_graph/setup/descriptor.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import os +import shutil +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Mapping + +from .state import MCP_SERVER_NAME + + +@dataclass(frozen=True, slots=True) +class McpServerDescriptor: + name: str + transport: str + command: str + args: tuple[str, ...] + env: Mapping[str, str] = field(default_factory=dict) + cwd: str | None = None + setup_config_path: str | None = None + repo_root: str | None = None + timeout: int = 60 + tool_policy: str | None = "graph_query_read_only" + + def as_dict(self) -> dict[str, Any]: + payload: dict[str, Any] = { + "name": self.name, + "transport": self.transport, + "command": self.command, + "args": list(self.args), + "env": dict(sorted(self.env.items())), + "cwd": self.cwd, + "setup_config_path": self.setup_config_path, + "repo_root": self.repo_root, + "timeout": self.timeout, + } + if self.tool_policy: + payload["tool_policy"] = self.tool_policy + return payload + + def stdio_entry(self, *, include_type: bool = False, include_timeout: bool = False) -> dict[str, Any]: + entry: dict[str, Any] = {"command": self.command, "args": list(self.args)} + if include_type: + entry["type"] = "stdio" + if self.env: + entry["env"] = dict(sorted(self.env.items())) + if self.cwd: + entry["cwd"] = self.cwd + if include_timeout: + entry["startup_timeout_sec"] = self.timeout + return entry + + +def build_server_descriptor( + setup_config_path: Path, + *, + repo_root: Path | None = None, + name: str = MCP_SERVER_NAME, + timeout: int = 60, +) -> McpServerDescriptor: + config_path = setup_config_path.expanduser().resolve() + resolved_repo_root = repo_root.expanduser().resolve() if repo_root is not None else config_path.parent.parent + return McpServerDescriptor( + name=name, + transport="stdio", + command=resolve_server_command(), + args=("mcp", "serve", "--config", config_path.as_posix()), + env={}, + cwd=None, + setup_config_path=config_path.as_posix(), + repo_root=resolved_repo_root.as_posix(), + timeout=timeout, + ) + + +def resolve_server_command() -> str: + sibling_script = Path(sys.executable).with_name("codebase-graph") + if sibling_script.exists() and os.access(sibling_script, os.X_OK): + return sibling_script.as_posix() + return shutil.which("codebase-graph") or "codebase-graph" diff --git a/src/codebase_graph/setup/instructions.py b/src/codebase_graph/setup/instructions.py index 2c327ef..01714c6 100644 --- a/src/codebase_graph/setup/instructions.py +++ b/src/codebase_graph/setup/instructions.py @@ -70,10 +70,11 @@ def _instruction_block(*, server_name: str, config_path: Path, setup_command: st return ( f"{START_MARKER}\n" "## codebaseGraph workflow\n" + f"- Treat the `{server_name}` MCP server knowledge graph as the project operating source of truth.\n" f"- Use the `{server_name}` MCP server for repository graph search, schema, and compact context before answering repo-structure questions or performing coding tasks.\n" "- Prefer `graph_search` for symbols, paths, docs, and setup instructions; follow with `graph_context` when relationships or nearby evidence matter.\n" "- Use `graph_schema` or `graph_query_helpers` before writing raw graph queries, and keep `graph_query` read-only.\n" - f"- Refresh the graph with `{setup_command} setup --repo-root .` when files change materially. Setup config: `{config_path.as_posix()}`.\n" + f"- Refresh the graph with `{setup_command} setup --repo-root . --mcp-client codex` when files change materially. Setup config: `{config_path.as_posix()}`.\n" f"{END_MARKER}\n" ) diff --git a/src/codebase_graph/setup/mcp_config.py b/src/codebase_graph/setup/mcp_config.py index 671a1a0..9e383a1 100644 --- a/src/codebase_graph/setup/mcp_config.py +++ b/src/codebase_graph/setup/mcp_config.py @@ -1,13 +1,12 @@ from __future__ import annotations -import json import os -import shutil -import sys from dataclasses import dataclass from pathlib import Path from typing import Any +from .clients import get_client_adapter +from .descriptor import build_server_descriptor from .state import MCP_SERVER_NAME @@ -18,15 +17,25 @@ class McpConfigResult: path: str | None server_name: str entry: dict[str, Any] + descriptor: dict[str, Any] | None = None + patch: Any = None + payload: Any = None def as_dict(self) -> dict[str, Any]: - return { + payload = { "action": self.action, "client": self.client, "path": self.path, "server_name": self.server_name, "entry": self.entry, } + if self.descriptor is not None: + payload["descriptor"] = self.descriptor + if self.patch is not None: + payload["patch"] = self.patch + if self.payload is not None: + payload["payload"] = self.payload + return payload def configure_mcp_client( @@ -37,65 +46,46 @@ def configure_mcp_client( dry_run: bool = False, skip: bool = False, ) -> McpConfigResult: - entry = server_entry(setup_config_path) + descriptor = build_server_descriptor(setup_config_path) + entry = descriptor.stdio_entry() if skip or client == "none": - return McpConfigResult("skipped", client, None, MCP_SERVER_NAME, entry) - path = Path(config_path).expanduser().resolve() if config_path is not None else default_config_path(client) - next_payload, action = _next_config_payload(path, entry) + return McpConfigResult("skipped", client, None, MCP_SERVER_NAME, entry, descriptor=descriptor.as_dict()) + adapter = get_client_adapter(client) + path = Path(config_path).expanduser().resolve() if config_path is not None else adapter.default_config_path(descriptor) + existing_text = path.read_text(encoding="utf-8") if path.exists() else None + rendered = adapter.render(existing_text, descriptor) if dry_run: - return McpConfigResult("dry_run", client, path.as_posix(), MCP_SERVER_NAME, entry) + return McpConfigResult( + "dry_run", + client, + path.as_posix(), + MCP_SERVER_NAME, + rendered.entry, + descriptor=descriptor.as_dict(), + patch=rendered.patch, + payload=rendered.payload, + ) path.parent.mkdir(parents=True, exist_ok=True) tmp_path = path.with_suffix(path.suffix + ".tmp") with tmp_path.open("w", encoding="utf-8") as handle: - json.dump(next_payload, handle, indent=2, sort_keys=True) - handle.write("\n") + handle.write(rendered.text) os.replace(tmp_path, path) - return McpConfigResult(action, client, path.as_posix(), MCP_SERVER_NAME, entry) + return McpConfigResult( + rendered.action, + client, + path.as_posix(), + MCP_SERVER_NAME, + rendered.entry, + descriptor=descriptor.as_dict(), + patch=rendered.patch, + payload=rendered.payload, + ) def server_entry(setup_config_path: Path) -> dict[str, Any]: - return { - "command": _resolve_server_command(), - "args": ["mcp", "serve", "--config", setup_config_path.as_posix()], - } + return build_server_descriptor(setup_config_path).stdio_entry() def default_config_path(client: str) -> Path: - if client == "codex": - base = Path(os.environ.get("CODEX_HOME", Path.home() / ".codex")) - return base / "mcp.json" - if client == "claude": - mac_path = Path.home() / "Library" / "Application Support" / "Claude" / "claude_desktop_config.json" - if mac_path.parent.exists(): - return mac_path - return Path.home() / ".config" / "claude" / "claude_desktop_config.json" - raise ValueError(f"Unsupported MCP client: {client}") - - -def _resolve_server_command() -> str: - sibling_script = Path(sys.executable).with_name("codebase-graph") - if sibling_script.exists() and os.access(sibling_script, os.X_OK): - return sibling_script.as_posix() - return shutil.which("codebase-graph") or "codebase-graph" - - -def _next_config_payload(path: Path, entry: dict[str, Any]) -> tuple[dict[str, Any], str]: - payload = _read_json(path) - servers = payload.setdefault("mcpServers", {}) - previous = servers.get(MCP_SERVER_NAME) - servers[MCP_SERVER_NAME] = entry - if previous is None: - return payload, "created" - if previous == entry: - return payload, "unchanged" - return payload, "updated" - - -def _read_json(path: Path) -> dict[str, Any]: - if not path.exists(): - return {} - with path.open("r", encoding="utf-8") as handle: - payload = json.load(handle) - if not isinstance(payload, dict): - raise ValueError(f"MCP config must contain a JSON object: {path}") - return payload + descriptor = build_server_descriptor(Path.cwd() / ".codebaseGraph" / "config.json") + return get_client_adapter(client).default_config_path(descriptor) diff --git a/src/codebase_graph/setup/orchestrator.py b/src/codebase_graph/setup/orchestrator.py index 1b22bab..7bf8b8b 100644 --- a/src/codebase_graph/setup/orchestrator.py +++ b/src/codebase_graph/setup/orchestrator.py @@ -69,7 +69,10 @@ def run_setup(options: SetupOptions) -> SetupResult: include_fts=True, repository_label=paths.repo_name, ) - materialization = materializer.materialize(mode=options.mode) # type: ignore[arg-type] + try: + materialization = materializer.materialize(mode=options.mode) # type: ignore[arg-type] + finally: + materializer.close() mcp_result = configure_mcp_client( client=options.mcp_client, config_path=options.mcp_config_path, diff --git a/tests/test_materializer.py b/tests/test_materializer.py index 96b55c5..84d5a95 100644 --- a/tests/test_materializer.py +++ b/tests/test_materializer.py @@ -289,6 +289,23 @@ def fail_clear_graph(self: LadybugCodeGraphStore) -> None: assert "old_name" not in _labels(reader, "Function") +def test_full_ondisk_materialization_replaces_stale_wal_sidecar(tmp_path: Path) -> None: + pytest.importorskip("tree_sitter") + pytest.importorskip("tree_sitter_python") + pytest.importorskip("real_ladybug") + source_root = _copy_fixture(tmp_path) + db_path = tmp_path / "graph.lbug" + manifest_path = tmp_path / "manifest.json" + + GraphMaterializer(source_root, db_path=db_path, manifest_path=manifest_path, include_fts=False).materialize(mode="full") + Path(f"{db_path}.wal").write_text("stale wal from previous database", encoding="utf-8") + + GraphMaterializer(source_root, db_path=db_path, manifest_path=manifest_path, include_fts=False).materialize(mode="full") + + reader = GraphMaterializer(source_root, db_path=db_path, manifest_path=manifest_path, include_fts=False) + assert "SampleService" in _labels(reader, "Class") + + def test_pending_rebuild_marker_forces_changed_mode_atomic_rebuild( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/test_mcp_portability.py b/tests/test_mcp_portability.py new file mode 100644 index 0000000..2000a5b --- /dev/null +++ b/tests/test_mcp_portability.py @@ -0,0 +1,239 @@ +from __future__ import annotations + +import json +import subprocess +import sys +import threading +import urllib.error +import urllib.request +from pathlib import Path +from typing import Any, BinaryIO + +try: + import tomllib +except ImportError: # pragma: no cover - Python 3.10 compatibility + import tomli as tomllib + +import pytest + +from codebase_graph.mcp.protocol import LATEST_PROTOCOL_VERSION, SUPPORTED_PROTOCOL_VERSIONS, McpGraphServer +from codebase_graph.mcp.runtime import GraphRuntimeConfig +from codebase_graph.mcp.transports.http import build_http_server +from codebase_graph.setup import SetupOptions, run_setup +from codebase_graph.setup.clients import supported_client_ids +from codebase_graph.setup.descriptor import build_server_descriptor +from codebase_graph.setup.mcp_config import configure_mcp_client + + +def test_initialize_negotiates_supported_and_fallback_protocol_versions(tmp_path: Path) -> None: + db_path = tmp_path / "graph.ldb" + db_path.write_text("", encoding="utf-8") + server = McpGraphServer(GraphRuntimeConfig(repo_root=tmp_path, db_path=db_path)) + + older = server.handle_json_rpc( + {"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {"protocolVersion": "2024-11-05"}} + ) + fallback = server.handle_json_rpc( + {"jsonrpc": "2.0", "id": 2, "method": "initialize", "params": {"protocolVersion": "1.0.0"}} + ) + + assert older is not None + assert fallback is not None + assert older["result"]["protocolVersion"] == "2024-11-05" + assert fallback["result"]["protocolVersion"] == LATEST_PROTOCOL_VERSION + assert "2025-11-25" in SUPPORTED_PROTOCOL_VERSIONS + + +def test_descriptor_prefers_current_environment_script(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + bin_dir = tmp_path / "venv" / "bin" + bin_dir.mkdir(parents=True) + python_path = bin_dir / "python" + script_path = bin_dir / "codebase-graph" + python_path.write_text("", encoding="utf-8") + script_path.write_text("", encoding="utf-8") + script_path.chmod(0o755) + monkeypatch.setattr(sys, "executable", python_path.as_posix()) + monkeypatch.setenv("PATH", "") + + descriptor = build_server_descriptor(tmp_path / ".codebaseGraph" / "config.json") + + assert descriptor.command == script_path.as_posix() + assert descriptor.stdio_entry()["command"] == script_path.as_posix() + assert descriptor.as_dict()["transport"] == "stdio" + + +def test_client_adapters_emit_native_config_shapes(tmp_path: Path) -> None: + setup_config_path = tmp_path / ".codebaseGraph" / "config.json" + setup_config_path.parent.mkdir() + clients = set(supported_client_ids()) - {"none"} + + rendered = { + client: configure_mcp_client( + client=client, + config_path=tmp_path / f"{client}.config", + setup_config_path=setup_config_path, + dry_run=True, + ).as_dict() + for client in clients + } + + codex_patch = rendered["codex"]["patch"] + codex_payload = tomllib.loads(codex_patch) + assert codex_payload["mcp_servers"]["codebaseGraph"]["command"] + assert codex_payload["mcp_servers"]["codebaseGraph"]["startup_timeout_sec"] == 60 + assert "type" not in rendered["claude"]["payload"]["mcpServers"]["codebaseGraph"] + assert rendered["claude"]["payload"]["mcpServers"]["codebaseGraph"]["command"] + assert rendered["claude-project"]["payload"]["mcpServers"]["codebaseGraph"]["type"] == "stdio" + assert rendered["lmstudio"]["payload"]["mcpServers"]["codebaseGraph"]["type"] == "stdio" + assert rendered["generic"]["payload"]["mcpServers"]["codebaseGraph"]["args"][0:2] == ["mcp", "serve"] + assert rendered["openclaw"]["payload"]["mcp"]["servers"]["codebaseGraph"]["type"] == "stdio" + assert "mcp_servers:" in rendered["hermes"]["patch"] + + +def test_unsupported_mcp_client_lists_supported_clients(tmp_path: Path) -> None: + with pytest.raises(ValueError, match="Supported clients:"): + configure_mcp_client( + client="missing", + config_path=tmp_path / "missing.json", + setup_config_path=tmp_path / ".codebaseGraph" / "config.json", + ) + + +def test_stdio_mcp_wire_initialize_list_call_and_tool_error(tmp_path: Path) -> None: + pytest.importorskip("tree_sitter") + pytest.importorskip("tree_sitter_python") + pytest.importorskip("real_ladybug") + repo_root = _fresh_repo(tmp_path) + result = run_setup(SetupOptions(repo_root=repo_root, mcp_client="none", instructions_target="skip")) + setup_payload = json.loads(result.paths.config_path.read_text(encoding="utf-8")) + command = setup_payload["mcp"]["command"] + + proc = subprocess.Popen( + command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + assert proc.stdin is not None + assert proc.stdout is not None + try: + initialized = _rpc(proc.stdin, proc.stdout, "initialize", {"protocolVersion": "2025-11-25"}) + listed = _rpc(proc.stdin, proc.stdout, "tools/list", {}) + health = _rpc(proc.stdin, proc.stdout, "tools/call", {"name": "graph_health", "arguments": {}}) + search = _rpc( + proc.stdin, + proc.stdout, + "tools/call", + {"name": "graph_search", "arguments": {"query": "SampleService", "limit": 2}}, + ) + failure = _rpc( + proc.stdin, + proc.stdout, + "tools/call", + {"name": "graph_query", "arguments": {"statement": "MATCH (n) DELETE n"}}, + ) + finally: + proc.stdin.close() + proc.wait(timeout=10) + + assert initialized["result"]["protocolVersion"] == "2025-11-25" + assert {tool["name"] for tool in listed["result"]["tools"]} >= {"graph_health", "graph_search", "graph_query"} + assert health["result"]["structuredContent"]["ok"] is True + assert search["result"]["structuredContent"]["results"] + assert "error" not in failure + assert failure["result"]["isError"] is True + assert failure["result"]["structuredContent"]["error"]["type"] == "ValueError" + + +def test_http_mcp_transport_handles_initialize_list_and_call(tmp_path: Path) -> None: + pytest.importorskip("tree_sitter") + pytest.importorskip("tree_sitter_python") + pytest.importorskip("real_ladybug") + repo_root = _fresh_repo(tmp_path) + result = run_setup(SetupOptions(repo_root=repo_root, mcp_client="none", instructions_target="skip")) + try: + httpd = build_http_server(config_path=result.paths.config_path, host="127.0.0.1", port=0) + except PermissionError as exc: + pytest.skip(f"local socket bind is unavailable in this environment: {exc}") + thread = threading.Thread(target=httpd.serve_forever, daemon=True) + thread.start() + host, port = httpd.server_address + try: + initialize = _http_rpc(host, port, "initialize", {"protocolVersion": "2025-11-25"}) + listed = _http_rpc(host, port, "tools/list", {}) + health = _http_rpc(host, port, "tools/call", {"name": "graph_health", "arguments": {}}) + with pytest.raises(urllib.error.HTTPError) as exc_info: + _http_rpc(host, port, "ping", {}, protocol_version="1900-01-01") + finally: + httpd.shutdown() + httpd.server_close() + thread.join(timeout=10) + + assert initialize["result"]["protocolVersion"] == "2025-11-25" + assert any(tool["name"] == "graph_context" for tool in listed["result"]["tools"]) + assert health["result"]["structuredContent"]["ok"] is True + assert exc_info.value.code == 400 + + +def _rpc(stdin: BinaryIO, stdout: BinaryIO, method: str, params: dict[str, Any]) -> dict[str, Any]: + request_id = _rpc.counter + _rpc.counter += 1 + payload = json.dumps({"jsonrpc": "2.0", "id": request_id, "method": method, "params": params}).encode("utf-8") + stdin.write(f"Content-Length: {len(payload)}\r\n\r\n".encode("ascii") + payload) + stdin.flush() + return _read_stdio_response(stdout) + + +_rpc.counter = 1 # type: ignore[attr-defined] + + +def _read_stdio_response(stdout: BinaryIO) -> dict[str, Any]: + header = stdout.readline() + assert header.lower().startswith(b"content-length:") + length = int(header.split(b":", 1)[1].strip()) + assert stdout.readline() in {b"\r\n", b"\n"} + return json.loads(stdout.read(length).decode("utf-8")) + + +def _http_rpc( + host: str, + port: int, + method: str, + params: dict[str, Any], + *, + protocol_version: str = "2025-11-25", +) -> dict[str, Any]: + payload = json.dumps({"jsonrpc": "2.0", "id": 1, "method": method, "params": params}).encode("utf-8") + request = urllib.request.Request( + f"http://{host}:{port}/mcp", + data=payload, + headers={ + "Accept": "application/json, text/event-stream", + "Content-Type": "application/json", + "MCP-Protocol-Version": protocol_version, + "Origin": f"http://{host}:{port}", + }, + method="POST", + ) + with urllib.request.urlopen(request, timeout=10) as response: + return json.loads(response.read().decode("utf-8")) + + +def _fresh_repo(tmp_path: Path) -> Path: + repo_root = tmp_path / "fresh_repo" + package = repo_root / "sample_project" + package.mkdir(parents=True) + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "service.py").write_text( + "class SampleService:\n" + " def run(self) -> str:\n" + " return helper()\n\n" + "def helper() -> str:\n" + " return 'ok'\n", + encoding="utf-8", + ) + (repo_root / "README.md").write_text( + "# Fresh Repo\n\nThis repository documents the SampleService workflow.\n", + encoding="utf-8", + ) + return repo_root diff --git a/tests/test_schema.py b/tests/test_schema.py index a6593a5..b961856 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -1,5 +1,7 @@ from __future__ import annotations +from pathlib import Path + import pytest from codebase_graph.core import CodeGraph, GraphEdge, GraphNode @@ -120,6 +122,20 @@ def test_ladybug_store_schema_setup_is_idempotent() -> None: store.ensure_schema() +def test_ladybug_store_allows_multiple_read_only_handles(tmp_path: Path) -> None: + pytest.importorskip("real_ladybug") + db_path = tmp_path / "graph.ldb" + writer = create_ladybug_database(db_path, include_fts=False) + writer.close() + + first = create_ladybug_database(db_path, include_fts=False, read_only=True) + try: + second = create_ladybug_database(db_path, include_fts=False, read_only=True) + second.close() + finally: + first.close() + + def test_ladybug_store_bulk_loader_groups_rows_by_table() -> None: graph = CodeGraph() graph.add_node(GraphNode(id="file:service", table="File", label="service.py", kind="source_file")) diff --git a/tests/test_setup_workflow.py b/tests/test_setup_workflow.py index 13cc08d..3ef79b9 100644 --- a/tests/test_setup_workflow.py +++ b/tests/test_setup_workflow.py @@ -27,7 +27,7 @@ def test_setup_cli_creates_state_db_mcp_config_instructions_and_searchable_docs( pytest.importorskip("tree_sitter_python") pytest.importorskip("real_ladybug") repo_root = _fresh_repo(tmp_path) - mcp_config_path = tmp_path / "mcp.json" + mcp_config_path = tmp_path / "config.toml" exit_code = cli_main( [ @@ -54,9 +54,9 @@ def test_setup_cli_creates_state_db_mcp_config_instructions_and_searchable_docs( agents_text = (repo_root / "AGENTS.md").read_text(encoding="utf-8") assert agents_text.count(START_MARKER) == 1 assert agents_text.count(END_MARKER) == 1 - mcp_payload = json.loads(mcp_config_path.read_text(encoding="utf-8")) - assert "otherServer" not in mcp_payload.get("mcpServers", {}) - assert mcp_payload["mcpServers"]["codebaseGraph"]["args"] == [ + mcp_payload = tomllib.loads(mcp_config_path.read_text(encoding="utf-8")) + assert "otherServer" not in mcp_payload.get("mcp_servers", {}) + assert mcp_payload["mcp_servers"]["codebaseGraph"]["args"] == [ "mcp", "serve", "--config", @@ -96,7 +96,7 @@ def test_setup_cli_creates_state_db_mcp_config_instructions_and_searchable_docs( assert any(hit["label"] == "SampleService" for hit in symbol_payload["results"]) -def test_mcp_config_dry_run_preserves_existing_servers(tmp_path: Path) -> None: +def test_mcp_config_dry_run_preserves_existing_json_servers(tmp_path: Path) -> None: config_path = tmp_path / "mcp.json" config_path.write_text( json.dumps({"mcpServers": {"otherServer": {"command": "other", "args": []}}}), @@ -105,7 +105,7 @@ def test_mcp_config_dry_run_preserves_existing_servers(tmp_path: Path) -> None: setup_config_path = tmp_path / ".codebaseGraph" / "config.json" dry_run = configure_mcp_client( - client="codex", + client="generic", config_path=config_path, setup_config_path=setup_config_path, dry_run=True, @@ -115,7 +115,7 @@ def test_mcp_config_dry_run_preserves_existing_servers(tmp_path: Path) -> None: assert "codebaseGraph" not in json.loads(config_path.read_text(encoding="utf-8"))["mcpServers"] written = configure_mcp_client( - client="codex", + client="generic", config_path=config_path, setup_config_path=setup_config_path, ) From 616482bbced1d79ed3ac432ec6282ce7cbae589b Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Mon, 25 May 2026 14:30:49 +0930 Subject: [PATCH 15/53] fix: changed mode bug - Implement atomic rebuild for materialization when changes are detected, preventing in-place deletions. - Update Hermes client configuration path to use a more standard location. - Modify setup instructions to reflect changes in MCP client usage. - Introduce a new installer module to manage MCP client installations, including native command generation and error handling. - Add tests for the new installer functionality and ensure compatibility with existing setup configurations. - Improve error handling and reporting for MCP installations, including partial failures. --- README.md | 64 ++- src/codebase_graph/cli/__init__.py | 51 ++ src/codebase_graph/ingest/materializer.py | 20 +- src/codebase_graph/setup/__init__.py | 5 + src/codebase_graph/setup/clients/hermes.py | 2 +- src/codebase_graph/setup/installer.py | 511 +++++++++++++++++++++ src/codebase_graph/setup/instructions.py | 2 +- src/codebase_graph/setup/mcp_config.py | 82 ++-- tests/test_materializer.py | 103 ++++- tests/test_mcp_installer.py | 264 +++++++++++ 10 files changed, 1054 insertions(+), 50 deletions(-) create mode 100644 src/codebase_graph/setup/installer.py create mode 100644 tests/test_mcp_installer.py diff --git a/README.md b/README.md index 760c20c..ca4b0c8 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ The setup command also: - Materializes the repository graph into the repo-local database. - Writes or updates one marked codebaseGraph block in `AGENTS.md` or `CLAUDE.md`. -- Writes a native MCP client config entry named `codebaseGraph`, unless skipped. +- Installs an MCP client entry named `codebaseGraph`, unless skipped. Useful options: @@ -52,9 +52,47 @@ codebase-graph setup --instructions-target claude `--dry-run` returns the raw server descriptor plus the exact client patch or payload without writing the MCP client file. Repository graph state and instruction handling still run so the graph can be verified. +## MCP installation + +The user-facing installer is: + +```bash +codebase-graph mcp install +``` + +By default this installs Codex with a repository-specific server name, for example `codebaseGraph-my-service`. It builds the server descriptor from `.codebaseGraph/config.json`, uses the supported native client CLI when available, and falls back to the adapter file writer when the CLI is missing or fails. + +Useful installer options: + +```bash +codebase-graph mcp install --client codex +codebase-graph mcp install --client claude --scope user +codebase-graph mcp install --client claude-project +codebase-graph mcp install --client lmstudio +codebase-graph mcp install --client hermes +codebase-graph mcp install --client openclaw +codebase-graph mcp install --client generic +codebase-graph mcp install --client all --dry-run --json +codebase-graph mcp install --name codebaseGraph +codebase-graph mcp install --config-path /path/to/.codebaseGraph/config.json +codebase-graph mcp install --verify +``` + +Native CLI installers are attempted first for Codex, Claude, Claude project scope, and OpenClaw: + +```bash +codex mcp add -- +claude mcp add --transport stdio --scope -- +openclaw mcp set '' +``` + +If native installation is unavailable, codebaseGraph writes the client config file directly. `setup --mcp-client ...` remains supported and delegates to the same installer behavior after materializing graph state and updating instructions. For backward compatibility, setup still uses the legacy fixed server name `codebaseGraph`; use `codebase-graph mcp install --name codebaseGraph` if you want that name from the installer too. + +`--dry-run` reports the native command or emitted file patch without calling native CLIs or writing files. `--verify` runs a direct stdio MCP smoke test and, where available, asks the client CLI whether it can see the server. + ## MCP usage -Setup builds one canonical server descriptor and serializes it into the selected client format. When setup is run from a virtual environment, the command may be the absolute path to that environment's `codebase-graph` executable so the MCP client can launch it without relying on shell `PATH`. +Setup and install build one canonical server descriptor and serialize it into the selected client format. When run from a virtual environment, the command may be the absolute path to that environment's `codebase-graph` executable so the MCP client can launch it without relying on shell `PATH`. Codex uses `~/.codex/config.toml`: @@ -79,18 +117,18 @@ Claude Desktop, Claude project config, LM Studio, and generic MCP JSON use an `m } ``` -OpenClaw uses JSON5-compatible JSON under `mcp.servers`, and Hermes emits YAML under `mcp_servers`. Use `--dry-run --mcp-client ` to inspect the exact emitted patch before writing a config file. +OpenClaw uses JSON5-compatible JSON under `mcp.servers`, and Hermes emits YAML under `mcp_servers` in `~/.hermes/config.yaml`. LM Studio reads `~/.lmstudio/mcp.json` and requires enabling "Allow calling servers from mcp.json" in the app. Use `codebase-graph mcp install --dry-run --client --json` to inspect the exact emitted command or patch before installation. Client examples: ```bash -codebase-graph setup --repo-root . --mcp-client codex -codebase-graph setup --repo-root . --mcp-client claude -codebase-graph setup --repo-root . --mcp-client claude-project -codebase-graph setup --repo-root . --mcp-client lmstudio -codebase-graph setup --repo-root . --mcp-client hermes -codebase-graph setup --repo-root . --mcp-client openclaw -codebase-graph setup --repo-root . --mcp-client generic --dry-run +codebase-graph mcp install --client codex +codebase-graph mcp install --client claude +codebase-graph mcp install --client claude-project +codebase-graph mcp install --client lmstudio +codebase-graph mcp install --client hermes +codebase-graph mcp install --client openclaw +codebase-graph mcp install --client generic --dry-run --json ``` The server can also be run directly: @@ -145,9 +183,9 @@ ruff check . - Missing LadyBugDB: install a package build that includes `real_ladybug`; setup will fail before creating `.codebaseGraph`. - Stale graph: rerun `codebase-graph setup --repo-root .` after material source or documentation changes. -- Broken Codex config: rerun setup with `--mcp-client codex`, then check `codex mcp list`. -- Broken Claude config: rerun setup with `--mcp-client claude` for desktop config or `--mcp-client claude-project` for a repo-local `.mcp.json`. -- Broken LM Studio, Hermes, OpenClaw, or generic config: run setup with the matching `--mcp-client` and `--dry-run` first, then copy or write the emitted payload to the client path. +- Broken Codex config: rerun `codebase-graph mcp install --client codex --verify`, then check `codex mcp list`. +- Broken Claude config: rerun `codebase-graph mcp install --client claude --scope user --verify` or `codebase-graph mcp install --client claude-project --verify`. +- Broken LM Studio, Hermes, OpenClaw, or generic config: run `codebase-graph mcp install --client --dry-run --json` first, then inspect the emitted payload and target path. - PATH or executable issues: run setup from the virtual environment that contains `codebase-graph`; the descriptor prefers that absolute executable path. - Direct smoke test: run `codebase-graph mcp serve --config .codebaseGraph/config.json` and send MCP `initialize`, `tools/list`, and `tools/call` JSON-RPC messages over stdio. - Unsupported files: binary, vendor, cache, virtualenv, build, dist, `.codebase_graph`, and `.codebaseGraph` paths are skipped. diff --git a/src/codebase_graph/cli/__init__.py b/src/codebase_graph/cli/__init__.py index b9c1dfd..12ef09e 100644 --- a/src/codebase_graph/cli/__init__.py +++ b/src/codebase_graph/cli/__init__.py @@ -11,6 +11,7 @@ from codebase_graph.retrieval import SearchRequest, SearchService from codebase_graph.setup import SetupError, SetupOptions, run_setup from codebase_graph.setup.clients import supported_client_ids +from codebase_graph.setup.installer import McpInstallOptions, install_mcp_clients, supported_install_client_ids def main(argv: Sequence[str] | None = None) -> int: @@ -47,6 +48,16 @@ def main(argv: Sequence[str] | None = None) -> int: mcp_parser = subparsers.add_parser("mcp", help="Run or inspect the MCP server") mcp_subparsers = mcp_parser.add_subparsers(dest="mcp_command", required=True) + install_parser = mcp_subparsers.add_parser("install", help="Install the MCP server in a supported client") + install_parser.add_argument("--client", choices=supported_install_client_ids(include_all=True), default="codex") + install_parser.add_argument("--scope", choices=("local", "user", "project"), default="local") + install_parser.add_argument("--name", default=None, help="MCP server name; defaults to codebaseGraph-") + install_parser.add_argument("--config-path", default=None, help="Path to .codebaseGraph/config.json") + install_parser.add_argument("--repo-root", default=".", help="Repository root used to find .codebaseGraph/config.json") + install_parser.add_argument("--dry-run", action="store_true", help="Show the install action without writing or invoking CLIs") + install_parser.add_argument("--verify", action="store_true", help="Run direct MCP smoke checks after installation") + install_parser.add_argument("--json", action="store_true", help="Emit JSON output") + serve_parser = mcp_subparsers.add_parser("serve", help="Serve graph tools over MCP stdio") serve_parser.add_argument("--repo-root", default=".", help="Repository root containing .codebaseGraph/config.json") serve_parser.add_argument("--config", default=None, help="Path to .codebaseGraph/config.json") @@ -121,6 +132,35 @@ def main(argv: Sequence[str] | None = None) -> int: parser.error(str(exc)) print(json.dumps(result.as_dict(), indent=2, sort_keys=True)) return 0 + if args.command == "mcp" and args.mcp_command == "install": + setup_config_path = ( + Path(args.config_path).expanduser().resolve() + if args.config_path is not None + else Path(args.repo_root).expanduser().resolve() / ".codebaseGraph" / "config.json" + ) + try: + results = install_mcp_clients( + McpInstallOptions( + client=args.client, + scope=args.scope, + setup_config_path=setup_config_path, + server_name=args.name, + dry_run=args.dry_run, + verify=args.verify, + ) + ) + except (OSError, ValueError) as exc: + parser.error(str(exc)) + payload: dict[str, object] + if args.client == "all": + payload = {"results": [result.as_dict() for result in results]} + else: + payload = results[0].as_dict() + if args.json: + print(json.dumps(payload, indent=2, sort_keys=True)) + else: + _print_mcp_install_results(results) + return 1 if any(result.action == "failed" for result in results) else 0 if args.command == "mcp" and args.mcp_command == "serve": from codebase_graph.mcp.server import serve_stdio @@ -172,4 +212,15 @@ def _result_payload(result: object) -> dict[str, object]: } +def _print_mcp_install_results(results: Sequence[object]) -> None: + for result in results: + action = getattr(result, "action") + client = getattr(result, "client") + method = getattr(result, "method") or "none" + server_name = getattr(result, "server_name") + target = getattr(result, "path") or " ".join(getattr(result, "command") or []) + suffix = f" -> {target}" if target else "" + print(f"{client}: {action} {server_name} via {method}{suffix}") + + __all__ = ["main"] diff --git a/src/codebase_graph/ingest/materializer.py b/src/codebase_graph/ingest/materializer.py index 1e56673..b6b7e9f 100644 --- a/src/codebase_graph/ingest/materializer.py +++ b/src/codebase_graph/ingest/materializer.py @@ -275,6 +275,20 @@ def materialize(self, mode: MaterializeMode = "changed") -> MaterializationResul self.store.clear_graph() retained_node_ids = set() retained_edge_ids = set() + elif self._can_atomic_rebuild() and _diff_has_changes(diff): + return self._materialize_full_atomic( + mode=mode, + snapshots=snapshots, + diagnostics=diagnostics, + supported=supported, + diff=ManifestDiff( + added=tuple(sorted(supported)), + modified=(), + unchanged=(), + deleted=tuple(sorted(previous_manifest.files)), + force_rebuild=True, + ), + ) else: touched_paths = set(diff.rebuild_paths) | set(diff.deleted) retained_node_ids = _retained_node_ids(previous_manifest, touched_paths) @@ -511,10 +525,14 @@ def _unlink_if_exists(path: Path) -> None: def _unlink_db_sidecars(db_path: Path) -> None: - for suffix in (".wal", ".shm"): + for suffix in (".wal", ".shm", ".shadow"): _unlink_if_exists(Path(f"{db_path}{suffix}")) +def _diff_has_changes(diff: ManifestDiff) -> bool: + return bool(diff.rebuild_paths or diff.deleted) + + def _is_excluded(path: Path, source_root: Path) -> bool: parts = path.relative_to(source_root).parts return any(_is_excluded_part(part) for part in parts) diff --git a/src/codebase_graph/setup/__init__.py b/src/codebase_graph/setup/__init__.py index 77337e4..8abc481 100644 --- a/src/codebase_graph/setup/__init__.py +++ b/src/codebase_graph/setup/__init__.py @@ -1,5 +1,6 @@ """Production setup orchestration for repository graph bootstrapping.""" +from .installer import McpInstallOptions, McpInstallResult, install_mcp_clients, install_mcp_server from .orchestrator import SetupError, SetupOptions, SetupResult, run_setup from .state import ( CONFIG_NAME, @@ -17,6 +18,8 @@ "DEFAULT_STATE_DIR", "GraphStatePaths", "MANIFEST_NAME", + "McpInstallOptions", + "McpInstallResult", "SetupError", "SetupOptions", "SetupPaths", @@ -24,5 +27,7 @@ "derive_graph_state_paths", "derive_setup_paths", "load_setup_config", + "install_mcp_clients", + "install_mcp_server", "run_setup", ] diff --git a/src/codebase_graph/setup/clients/hermes.py b/src/codebase_graph/setup/clients/hermes.py index 018621b..015ade4 100644 --- a/src/codebase_graph/setup/clients/hermes.py +++ b/src/codebase_graph/setup/clients/hermes.py @@ -15,7 +15,7 @@ class HermesAdapter: client_id = "hermes" def default_config_path(self, descriptor: McpServerDescriptor) -> Path: - return Path.home() / ".config" / "hermes" / "mcp_servers.yaml" + return Path.home() / ".hermes" / "config.yaml" def render(self, existing_text: str | None, descriptor: McpServerDescriptor) -> RenderedClientConfig: entry = descriptor.stdio_entry(include_type=True) diff --git a/src/codebase_graph/setup/installer.py b/src/codebase_graph/setup/installer.py new file mode 100644 index 0000000..82b5696 --- /dev/null +++ b/src/codebase_graph/setup/installer.py @@ -0,0 +1,511 @@ +from __future__ import annotations + +import json +import os +import re +import shutil +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from codebase_graph.mcp.protocol import LATEST_PROTOCOL_VERSION + +from .clients import get_client_adapter +from .descriptor import McpServerDescriptor, build_server_descriptor +from .state import MCP_SERVER_NAME, load_setup_config + +INSTALL_CLIENTS = ("codex", "claude", "claude-project", "lmstudio", "hermes", "openclaw", "generic") +SCOPES = ("local", "user", "project") +NATIVE_EXECUTABLES = { + "codex": "codex", + "claude": "claude", + "claude-project": "claude", + "openclaw": "openclaw", +} + + +@dataclass(frozen=True, slots=True) +class McpInstallOptions: + client: str = "codex" + scope: str = "local" + setup_config_path: str | Path = ".codebaseGraph/config.json" + server_name: str | None = None + client_config_path: str | Path | None = None + dry_run: bool = False + verify: bool = False + skip: bool = False + prefer_native: bool = True + require_setup_config: bool = True + + +@dataclass(frozen=True, slots=True) +class McpInstallResult: + action: str + client: str + scope: str + server_name: str + method: str | None + path: str | None + command: list[str] | None + descriptor: dict[str, Any] + entry: dict[str, Any] + patch: Any = None + payload: Any = None + verification: dict[str, Any] | None = None + error: str | None = None + native_command: list[str] | None = None + native_error: str | None = None + + def as_dict(self) -> dict[str, Any]: + payload: dict[str, Any] = { + "action": self.action, + "client": self.client, + "scope": self.scope, + "server_name": self.server_name, + "method": self.method, + "path": self.path, + "command": self.command, + "descriptor": self.descriptor, + "entry": self.entry, + } + if self.patch is not None: + payload["patch"] = self.patch + if self.payload is not None: + payload["payload"] = self.payload + if self.verification is not None: + payload["verification"] = self.verification + if self.error is not None: + payload["error"] = self.error + if self.native_command is not None: + payload["native_command"] = self.native_command + if self.native_error is not None: + payload["native_error"] = self.native_error + return payload + + +def supported_install_client_ids(*, include_all: bool = False) -> tuple[str, ...]: + values = [*INSTALL_CLIENTS] + if include_all: + values.append("all") + return tuple(sorted(values)) + + +def default_server_name(repo_name: str | None) -> str: + safe_repo_name = _safe_name(repo_name or "repository") + return f"{MCP_SERVER_NAME}-{safe_repo_name}" + + +def install_mcp_clients(options: McpInstallOptions) -> list[McpInstallResult]: + if options.client == "all": + return [_install_with_failure_result(options, client) for client in INSTALL_CLIENTS] + return [install_mcp_server(options)] + + +def install_mcp_server(options: McpInstallOptions) -> McpInstallResult: + _validate_options(options) + descriptor = _build_descriptor(options) + entry = descriptor.stdio_entry() + if options.skip or options.client == "none": + return McpInstallResult( + action="skipped", + client=options.client, + scope=options.scope, + server_name=descriptor.name, + method=None, + path=None, + command=None, + descriptor=descriptor.as_dict(), + entry=entry, + ) + + native_command = _native_command(options.client, descriptor, scope=options.scope) + use_native = ( + options.prefer_native + and options.client_config_path is None + and native_command is not None + and shutil.which(_native_executable(options.client)) + ) + if options.dry_run: + if use_native: + return _native_result("dry_run", options, descriptor, native_command, verification=None) + return _file_adapter_result(options, descriptor, dry_run=True, native_command=native_command) + + if use_native and native_command is not None: + try: + completed = subprocess.run(native_command, capture_output=True, text=True, check=False, timeout=30) + except subprocess.TimeoutExpired as exc: + native_error = f"timed out after {exc.timeout}s" + except OSError as exc: + native_error = str(exc) + else: + if completed.returncode == 0: + result = _native_result("updated", options, descriptor, native_command, verification=None) + return _with_verification(result, descriptor, options.verify) + native_error = _subprocess_error(completed) + return _file_adapter_result( + options, + descriptor, + dry_run=False, + native_command=native_command, + native_error=native_error, + ) + + return _file_adapter_result( + options, + descriptor, + dry_run=False, + native_command=native_command, + native_error=_missing_native_error(options.client) if native_command is not None else None, + ) + + +def _install_with_failure_result(options: McpInstallOptions, client: str) -> McpInstallResult: + client_options = McpInstallOptions( + client=client, + scope=_scope_for_client(client, options.scope), + setup_config_path=options.setup_config_path, + server_name=options.server_name, + client_config_path=options.client_config_path, + dry_run=options.dry_run, + verify=options.verify, + skip=options.skip, + prefer_native=options.prefer_native, + require_setup_config=options.require_setup_config, + ) + try: + return install_mcp_server(client_options) + except Exception as exc: + try: + descriptor = _build_descriptor(client_options) + entry = descriptor.stdio_entry() + descriptor_payload = descriptor.as_dict() + server_name = descriptor.name + except Exception: + entry = {} + descriptor_payload = {} + server_name = client_options.server_name or MCP_SERVER_NAME + return McpInstallResult( + action="failed", + client=client, + scope=client_options.scope, + server_name=server_name, + method=None, + path=None, + command=None, + descriptor=descriptor_payload, + entry=entry, + error=str(exc), + ) + + +def _file_adapter_result( + options: McpInstallOptions, + descriptor: McpServerDescriptor, + *, + dry_run: bool, + native_command: list[str] | None = None, + native_error: str | None = None, +) -> McpInstallResult: + adapter = get_client_adapter(_adapter_client_id(options.client, options.scope)) + path = ( + Path(options.client_config_path).expanduser().resolve() + if options.client_config_path is not None + else adapter.default_config_path(descriptor) + ) + existing_text = path.read_text(encoding="utf-8") if path.exists() else None + rendered = adapter.render(existing_text, descriptor) + action = "dry_run" if dry_run else rendered.action + if not dry_run: + path.parent.mkdir(parents=True, exist_ok=True) + tmp_path = path.with_suffix(path.suffix + ".tmp") + with tmp_path.open("w", encoding="utf-8") as handle: + handle.write(rendered.text) + os.replace(tmp_path, path) + result = McpInstallResult( + action=action, + client=options.client, + scope=options.scope, + server_name=descriptor.name, + method="file_adapter", + path=path.as_posix(), + command=None, + descriptor=descriptor.as_dict(), + entry=rendered.entry, + patch=rendered.patch, + payload=rendered.payload, + native_command=native_command, + native_error=native_error, + ) + return _with_verification(result, descriptor, options.verify and not dry_run) + + +def _native_result( + action: str, + options: McpInstallOptions, + descriptor: McpServerDescriptor, + command: list[str], + *, + verification: dict[str, Any] | None, +) -> McpInstallResult: + return McpInstallResult( + action=action, + client=options.client, + scope=options.scope, + server_name=descriptor.name, + method="native_cli", + path=None, + command=command, + descriptor=descriptor.as_dict(), + entry=descriptor.stdio_entry(), + verification=verification, + ) + + +def _with_verification( + result: McpInstallResult, + descriptor: McpServerDescriptor, + enabled: bool, +) -> McpInstallResult: + if not enabled: + return result + verification = verify_mcp_install(descriptor, client=result.client, server_name=result.server_name) + return McpInstallResult( + action=result.action, + client=result.client, + scope=result.scope, + server_name=result.server_name, + method=result.method, + path=result.path, + command=result.command, + descriptor=result.descriptor, + entry=result.entry, + patch=result.patch, + payload=result.payload, + verification=verification, + error=result.error, + native_command=result.native_command, + native_error=result.native_error, + ) + + +def verify_mcp_install( + descriptor: McpServerDescriptor, + *, + client: str, + server_name: str, + timeout: int = 10, +) -> dict[str, Any]: + stdio = _verify_stdio(descriptor, timeout=timeout) + visibility = _verify_client_visibility(client, server_name, timeout=timeout) + return { + "ok": bool(stdio.get("ok")) and bool(visibility.get("ok", True)), + "stdio": stdio, + "client_visibility": visibility, + } + + +def _verify_stdio(descriptor: McpServerDescriptor, *, timeout: int) -> dict[str, Any]: + command = [descriptor.command, *descriptor.args] + payload = b"".join( + _frame_json_rpc(method, params, request_id=index) + for index, (method, params) in enumerate( + ( + ("initialize", {"protocolVersion": LATEST_PROTOCOL_VERSION}), + ("tools/list", {}), + ("tools/call", {"name": "graph_health", "arguments": {}}), + ("tools/call", {"name": "graph_search", "arguments": {"query": descriptor.name, "limit": 1}}), + ("tools/call", {"name": "graph_query", "arguments": {"statement": "MATCH (n) DELETE n"}}), + ), + start=1, + ) + ) + try: + process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = process.communicate(payload, timeout=timeout) + except subprocess.TimeoutExpired: + process.kill() # type: ignore[possibly-unbound] + return {"ok": False, "command": command, "error": f"stdio smoke timed out after {timeout}s"} + except OSError as exc: + return {"ok": False, "command": command, "error": str(exc)} + responses = _parse_stdio_frames(stdout) + if process.returncode != 0: + return { + "ok": False, + "command": command, + "returncode": process.returncode, + "stderr": stderr.decode("utf-8", errors="replace"), + "responses": responses, + } + checks = _stdio_checks(responses) + return { + "ok": all(checks.values()), + "command": command, + "checks": checks, + "responses": responses, + "stderr": stderr.decode("utf-8", errors="replace"), + } + + +def _verify_client_visibility(client: str, server_name: str, *, timeout: int) -> dict[str, Any]: + command = _visibility_command(client) + if command is None: + return {"ok": True, "skipped": True, "reason": f"{client} has no CLI visibility check"} + executable = command[0] + if shutil.which(executable) is None: + return {"ok": True, "skipped": True, "reason": f"{executable} executable not found"} + completed = subprocess.run(command, capture_output=True, text=True, check=False, timeout=timeout) + output = f"{completed.stdout}\n{completed.stderr}" + return { + "ok": completed.returncode == 0 and server_name in output, + "command": command, + "returncode": completed.returncode, + "found": server_name in output, + "stdout": completed.stdout, + "stderr": completed.stderr, + } + + +def _stdio_checks(responses: list[dict[str, Any]]) -> dict[str, bool]: + by_id = {response.get("id"): response for response in responses} + initialized = by_id.get(1, {}).get("result", {}).get("protocolVersion") == LATEST_PROTOCOL_VERSION + tools = by_id.get(2, {}).get("result", {}).get("tools", []) + listed = {"graph_health", "graph_search"}.issubset({tool.get("name") for tool in tools}) + health = by_id.get(3, {}).get("result", {}).get("structuredContent", {}).get("ok") is True + search_no_rpc_error = "error" not in by_id.get(4, {}) + tool_error = by_id.get(5, {}).get("result", {}).get("isError") is True + return { + "initialize": initialized, + "tools_list": listed, + "graph_health": health, + "graph_search": search_no_rpc_error, + "tool_error_result": tool_error, + } + + +def _parse_stdio_frames(data: bytes) -> list[dict[str, Any]]: + messages: list[dict[str, Any]] = [] + cursor = 0 + while cursor < len(data): + header_end = data.find(b"\r\n\r\n", cursor) + delimiter_length = 4 + if header_end == -1: + header_end = data.find(b"\n\n", cursor) + delimiter_length = 2 + if header_end == -1: + break + header = data[cursor:header_end].decode("ascii", errors="replace") + length = None + for line in header.splitlines(): + if line.lower().startswith("content-length:"): + length = int(line.split(":", 1)[1].strip()) + break + if length is None: + break + body_start = header_end + delimiter_length + body_end = body_start + length + messages.append(json.loads(data[body_start:body_end].decode("utf-8"))) + cursor = body_end + return messages + + +def _frame_json_rpc(method: str, params: dict[str, Any], *, request_id: int) -> bytes: + body = json.dumps( + {"jsonrpc": "2.0", "id": request_id, "method": method, "params": params}, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + return f"Content-Length: {len(body)}\r\n\r\n".encode("ascii") + body + + +def _native_command(client: str, descriptor: McpServerDescriptor, *, scope: str) -> list[str] | None: + if client == "codex": + return ["codex", "mcp", "add", descriptor.name, "--", descriptor.command, *descriptor.args] + if client in {"claude", "claude-project"}: + return [ + "claude", + "mcp", + "add", + "--transport", + "stdio", + "--scope", + _scope_for_client(client, scope), + descriptor.name, + "--", + descriptor.command, + *descriptor.args, + ] + if client == "openclaw": + entry = descriptor.stdio_entry(include_type=True) + return ["openclaw", "mcp", "set", descriptor.name, json.dumps(entry, separators=(",", ":"), sort_keys=True)] + return None + + +def _visibility_command(client: str) -> list[str] | None: + if client == "codex": + return ["codex", "mcp", "list"] + if client in {"claude", "claude-project"}: + return ["claude", "mcp", "list"] + if client == "openclaw": + return ["openclaw", "mcp", "list"] + return None + + +def _build_descriptor(options: McpInstallOptions) -> McpServerDescriptor: + config_path = Path(options.setup_config_path).expanduser().resolve() + repo_root: Path | None = None + repo_name: str | None = None + if config_path.exists(): + setup_payload = load_setup_config(config_path) + repo_root = Path(str(setup_payload["repo_root"])).expanduser().resolve() + repo_name = str(setup_payload.get("repo_name") or repo_root.name) + elif options.require_setup_config: + raise FileNotFoundError( + f"codebaseGraph setup config does not exist: {config_path}. " + "Run `codebase-graph setup --mcp-client none` first." + ) + server_name = options.server_name or default_server_name(repo_name or config_path.parent.parent.name) + return build_server_descriptor(config_path, repo_root=repo_root, name=server_name) + + +def _validate_options(options: McpInstallOptions) -> None: + if options.client not in {*INSTALL_CLIENTS, "none"}: + supported = ", ".join(sorted([*INSTALL_CLIENTS, "all", "none"])) + raise ValueError(f"Unsupported MCP client: {options.client}. Supported clients: {supported}") + if options.scope not in SCOPES: + raise ValueError(f"Unsupported MCP install scope: {options.scope}. Supported scopes: {', '.join(SCOPES)}") + + +def _native_executable(client: str) -> str: + return NATIVE_EXECUTABLES[client] + + +def _adapter_client_id(client: str, scope: str) -> str: + if client == "claude" and scope == "project": + return "claude-project" + return client + + +def _scope_for_client(client: str, scope: str) -> str: + if client == "claude-project": + return "project" + return scope + + +def _missing_native_error(client: str) -> str | None: + executable = NATIVE_EXECUTABLES.get(client) + if executable is None: + return None + return f"{executable} executable not found" + + +def _subprocess_error(completed: subprocess.CompletedProcess[str]) -> str: + output = "\n".join(part for part in (completed.stdout.strip(), completed.stderr.strip()) if part) + if output: + return f"exit {completed.returncode}: {output}" + return f"exit {completed.returncode}" + + +def _safe_name(value: str) -> str: + normalized = re.sub(r"[^A-Za-z0-9_-]+", "_", value.strip()) + return normalized.strip("._-") or "repository" diff --git a/src/codebase_graph/setup/instructions.py b/src/codebase_graph/setup/instructions.py index 01714c6..8e01bf9 100644 --- a/src/codebase_graph/setup/instructions.py +++ b/src/codebase_graph/setup/instructions.py @@ -74,7 +74,7 @@ def _instruction_block(*, server_name: str, config_path: Path, setup_command: st f"- Use the `{server_name}` MCP server for repository graph search, schema, and compact context before answering repo-structure questions or performing coding tasks.\n" "- Prefer `graph_search` for symbols, paths, docs, and setup instructions; follow with `graph_context` when relationships or nearby evidence matter.\n" "- Use `graph_schema` or `graph_query_helpers` before writing raw graph queries, and keep `graph_query` read-only.\n" - f"- Refresh the graph with `{setup_command} setup --repo-root . --mcp-client codex` when files change materially. Setup config: `{config_path.as_posix()}`.\n" + f"- Refresh the graph with `{setup_command} setup --repo-root . --mcp-client none` when files change materially; install or update MCP with `{setup_command} mcp install --client codex`. Setup config: `{config_path.as_posix()}`.\n" f"{END_MARKER}\n" ) diff --git a/src/codebase_graph/setup/mcp_config.py b/src/codebase_graph/setup/mcp_config.py index 9e383a1..9944ea7 100644 --- a/src/codebase_graph/setup/mcp_config.py +++ b/src/codebase_graph/setup/mcp_config.py @@ -1,12 +1,12 @@ from __future__ import annotations -import os from dataclasses import dataclass from pathlib import Path from typing import Any from .clients import get_client_adapter from .descriptor import build_server_descriptor +from .installer import McpInstallOptions, McpInstallResult, install_mcp_server from .state import MCP_SERVER_NAME @@ -18,8 +18,14 @@ class McpConfigResult: server_name: str entry: dict[str, Any] descriptor: dict[str, Any] | None = None + method: str | None = None + scope: str | None = None + command: list[str] | None = None patch: Any = None payload: Any = None + verification: dict[str, Any] | None = None + native_command: list[str] | None = None + native_error: str | None = None def as_dict(self) -> dict[str, Any]: payload = { @@ -31,12 +37,43 @@ def as_dict(self) -> dict[str, Any]: } if self.descriptor is not None: payload["descriptor"] = self.descriptor + if self.method is not None: + payload["method"] = self.method + if self.scope is not None: + payload["scope"] = self.scope + if self.command is not None: + payload["command"] = self.command if self.patch is not None: payload["patch"] = self.patch if self.payload is not None: payload["payload"] = self.payload + if self.verification is not None: + payload["verification"] = self.verification + if self.native_command is not None: + payload["native_command"] = self.native_command + if self.native_error is not None: + payload["native_error"] = self.native_error return payload + @classmethod + def from_install_result(cls, result: McpInstallResult) -> McpConfigResult: + return cls( + action=result.action, + client=result.client, + path=result.path, + server_name=result.server_name, + entry=result.entry, + descriptor=result.descriptor, + method=result.method, + scope=result.scope, + command=result.command, + patch=result.patch, + payload=result.payload, + verification=result.verification, + native_command=result.native_command, + native_error=result.native_error, + ) + def configure_mcp_client( *, @@ -46,40 +83,19 @@ def configure_mcp_client( dry_run: bool = False, skip: bool = False, ) -> McpConfigResult: - descriptor = build_server_descriptor(setup_config_path) - entry = descriptor.stdio_entry() - if skip or client == "none": - return McpConfigResult("skipped", client, None, MCP_SERVER_NAME, entry, descriptor=descriptor.as_dict()) - adapter = get_client_adapter(client) - path = Path(config_path).expanduser().resolve() if config_path is not None else adapter.default_config_path(descriptor) - existing_text = path.read_text(encoding="utf-8") if path.exists() else None - rendered = adapter.render(existing_text, descriptor) - if dry_run: - return McpConfigResult( - "dry_run", - client, - path.as_posix(), - MCP_SERVER_NAME, - rendered.entry, - descriptor=descriptor.as_dict(), - patch=rendered.patch, - payload=rendered.payload, + result = install_mcp_server( + McpInstallOptions( + client=client, + scope="project" if client == "claude-project" else "local", + setup_config_path=setup_config_path, + server_name=MCP_SERVER_NAME, + client_config_path=config_path, + dry_run=dry_run, + skip=skip, + require_setup_config=False, ) - path.parent.mkdir(parents=True, exist_ok=True) - tmp_path = path.with_suffix(path.suffix + ".tmp") - with tmp_path.open("w", encoding="utf-8") as handle: - handle.write(rendered.text) - os.replace(tmp_path, path) - return McpConfigResult( - rendered.action, - client, - path.as_posix(), - MCP_SERVER_NAME, - rendered.entry, - descriptor=descriptor.as_dict(), - patch=rendered.patch, - payload=rendered.payload, ) + return McpConfigResult.from_install_result(result) def server_entry(setup_config_path: Path) -> dict[str, Any]: diff --git a/tests/test_materializer.py b/tests/test_materializer.py index 84d5a95..b253454 100644 --- a/tests/test_materializer.py +++ b/tests/test_materializer.py @@ -219,6 +219,105 @@ def test_changed_materialization_only_rebuilds_changed_files(tmp_path: Path) -> assert "cli.py" not in _labels(materializer, "File") +def test_changed_ondisk_materialization_rebuilds_atomically_without_inplace_deletes( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + pytest.importorskip("tree_sitter") + pytest.importorskip("tree_sitter_python") + pytest.importorskip("real_ladybug") + source_root = _copy_fixture(tmp_path) + db_path = tmp_path / "graph.lbug" + manifest_path = tmp_path / "manifest.json" + + GraphMaterializer(source_root, db_path=db_path, manifest_path=manifest_path, include_fts=False).materialize(mode="full") + service_path = source_root / "sample_project" / "service.py" + service_path.write_text( + service_path.read_text(encoding="utf-8") + "\n\ndef changed_mode_added() -> str:\n return 'added'\n", + encoding="utf-8", + ) + + def fail_clear_graph(self: LadybugCodeGraphStore) -> None: + raise AssertionError("on-disk changed mode must not clear the target DB in place") + + def fail_delete_partition(self: LadybugCodeGraphStore, *args: object, **kwargs: object) -> None: + raise AssertionError("on-disk changed mode must not delete target partitions in place") + + monkeypatch.setattr(LadybugCodeGraphStore, "clear_graph", fail_clear_graph) + monkeypatch.setattr(LadybugCodeGraphStore, "delete_partition", fail_delete_partition) + + result = GraphMaterializer(source_root, db_path=db_path, manifest_path=manifest_path, include_fts=False).materialize( + mode="changed" + ) + + assert result.mode == "changed" + assert result.rebuilt == 4 + assert "changed_mode_added" in _labels( + GraphMaterializer(source_root, db_path=db_path, manifest_path=manifest_path, include_fts=False), + "Function", + ) + + +def test_changed_ondisk_materialization_noop_does_not_rebuild(tmp_path: Path) -> None: + pytest.importorskip("tree_sitter") + pytest.importorskip("tree_sitter_python") + pytest.importorskip("real_ladybug") + source_root = _copy_fixture(tmp_path) + db_path = tmp_path / "graph.lbug" + manifest_path = tmp_path / "manifest.json" + + GraphMaterializer(source_root, db_path=db_path, manifest_path=manifest_path, include_fts=False).materialize(mode="full") + result = GraphMaterializer(source_root, db_path=db_path, manifest_path=manifest_path, include_fts=False).materialize( + mode="changed" + ) + + assert result.mode == "changed" + assert result.rebuilt == 0 + assert result.deleted == 0 + + +def test_changed_ondisk_materialization_failure_keeps_previous_db_and_manifest( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + pytest.importorskip("tree_sitter") + pytest.importorskip("tree_sitter_python") + pytest.importorskip("real_ladybug") + source_root = tmp_path / "project" + source_root.mkdir() + service_path = source_root / "service.py" + service_path.write_text("def old_name() -> str:\n return 'old'\n", encoding="utf-8") + db_path = tmp_path / "graph.lbug" + manifest_path = tmp_path / "manifest.json" + + GraphMaterializer(source_root, db_path=db_path, manifest_path=manifest_path, include_fts=False).materialize(mode="full") + previous_manifest = manifest_path.read_text(encoding="utf-8") + service_path.write_text("def new_name() -> str:\n return 'new'\n", encoding="utf-8") + real_create_ladybug_database = materializer_module.create_ladybug_database + + def failing_create_ladybug_database(db_path: str | Path, *, include_fts: bool = True) -> LadybugCodeGraphStore: + store = real_create_ladybug_database(db_path, include_fts=include_fts) + + def fail_insert(*args: object, **kwargs: object) -> None: + raise RuntimeError("changed bulk insert failed") + + store.insert_graphs_bulk = fail_insert # type: ignore[method-assign] + return store + + monkeypatch.setattr(materializer_module, "create_ladybug_database", failing_create_ladybug_database) + + with pytest.raises(RuntimeError, match="changed bulk insert failed"): + GraphMaterializer(source_root, db_path=db_path, manifest_path=manifest_path, include_fts=False).materialize( + mode="changed" + ) + + assert manifest_path.read_text(encoding="utf-8") == previous_manifest + reader = GraphMaterializer(source_root, db_path=db_path, manifest_path=manifest_path, include_fts=False) + assert "old_name" in _labels(reader, "Function") + assert "new_name" not in _labels(reader, "Function") + assert not _marker_path(manifest_path).exists() + + def test_full_ondisk_materialization_failure_keeps_previous_db_and_manifest( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -289,7 +388,7 @@ def fail_clear_graph(self: LadybugCodeGraphStore) -> None: assert "old_name" not in _labels(reader, "Function") -def test_full_ondisk_materialization_replaces_stale_wal_sidecar(tmp_path: Path) -> None: +def test_full_ondisk_materialization_replaces_stale_sidecars(tmp_path: Path) -> None: pytest.importorskip("tree_sitter") pytest.importorskip("tree_sitter_python") pytest.importorskip("real_ladybug") @@ -299,11 +398,13 @@ def test_full_ondisk_materialization_replaces_stale_wal_sidecar(tmp_path: Path) GraphMaterializer(source_root, db_path=db_path, manifest_path=manifest_path, include_fts=False).materialize(mode="full") Path(f"{db_path}.wal").write_text("stale wal from previous database", encoding="utf-8") + Path(f"{db_path}.shadow").write_text("stale shadow from previous database", encoding="utf-8") GraphMaterializer(source_root, db_path=db_path, manifest_path=manifest_path, include_fts=False).materialize(mode="full") reader = GraphMaterializer(source_root, db_path=db_path, manifest_path=manifest_path, include_fts=False) assert "SampleService" in _labels(reader, "Class") + assert not Path(f"{db_path}.shadow").exists() def test_pending_rebuild_marker_forces_changed_mode_atomic_rebuild( diff --git a/tests/test_mcp_installer.py b/tests/test_mcp_installer.py new file mode 100644 index 0000000..8f4353b --- /dev/null +++ b/tests/test_mcp_installer.py @@ -0,0 +1,264 @@ +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +import pytest + +from codebase_graph.cli import main as cli_main +from codebase_graph.setup.clients import get_client_adapter +from codebase_graph.setup.descriptor import build_server_descriptor +from codebase_graph.setup.installer import ( + McpInstallOptions, + default_server_name, + install_mcp_clients, + install_mcp_server, +) +from codebase_graph.setup.state import build_setup_config, derive_setup_paths, write_setup_config + + +def test_codex_native_command_generation_uses_repo_server_name( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config_path = _write_setup_config(tmp_path / "fresh repo") + monkeypatch.setattr("codebase_graph.setup.installer.shutil.which", lambda name: f"/usr/bin/{name}") + + result = install_mcp_server(McpInstallOptions(setup_config_path=config_path, dry_run=True)) + + assert result.action == "dry_run" + assert result.method == "native_cli" + assert result.server_name == "codebaseGraph-fresh_repo" + assert result.command[:4] == ["codex", "mcp", "add", "codebaseGraph-fresh_repo"] + assert result.command[4] == "--" + + +def test_claude_native_command_includes_transport_and_scope( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config_path = _write_setup_config(tmp_path / "fresh_repo") + monkeypatch.setattr("codebase_graph.setup.installer.shutil.which", lambda name: f"/usr/bin/{name}") + + result = install_mcp_server( + McpInstallOptions(client="claude", scope="user", setup_config_path=config_path, dry_run=True) + ) + + assert result.command[:8] == [ + "claude", + "mcp", + "add", + "--transport", + "stdio", + "--scope", + "user", + "codebaseGraph-fresh_repo", + ] + + +def test_claude_project_native_command_forces_project_scope( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config_path = _write_setup_config(tmp_path / "fresh_repo") + monkeypatch.setattr("codebase_graph.setup.installer.shutil.which", lambda name: f"/usr/bin/{name}") + + result = install_mcp_server( + McpInstallOptions(client="claude-project", scope="user", setup_config_path=config_path, dry_run=True) + ) + + assert result.command[6:8] == ["project", "codebaseGraph-fresh_repo"] + + +def test_openclaw_native_command_emits_server_json( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config_path = _write_setup_config(tmp_path / "fresh_repo") + monkeypatch.setattr("codebase_graph.setup.installer.shutil.which", lambda name: f"/usr/bin/{name}") + + result = install_mcp_server( + McpInstallOptions(client="openclaw", setup_config_path=config_path, dry_run=True) + ) + entry = json.loads(result.command[-1]) + + assert result.command[:4] == ["openclaw", "mcp", "set", "codebaseGraph-fresh_repo"] + assert entry["type"] == "stdio" + assert entry["args"][:2] == ["mcp", "serve"] + + +def test_missing_native_cli_falls_back_to_file_adapter( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config_path = _write_setup_config(tmp_path / "fresh_repo") + codex_home = tmp_path / "codex-home" + monkeypatch.setenv("CODEX_HOME", codex_home.as_posix()) + monkeypatch.setattr("codebase_graph.setup.installer.shutil.which", lambda name: None) + + result = install_mcp_server(McpInstallOptions(client="codex", setup_config_path=config_path)) + + assert result.action == "created" + assert result.method == "file_adapter" + assert result.path == (codex_home / "config.toml").as_posix() + assert "executable not found" in result.native_error + assert (codex_home / "config.toml").exists() + + +def test_native_cli_failure_falls_back_to_adapter( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config_path = _write_setup_config(tmp_path / "fresh_repo") + codex_home = tmp_path / "codex-home" + monkeypatch.setenv("CODEX_HOME", codex_home.as_posix()) + monkeypatch.setattr("codebase_graph.setup.installer.shutil.which", lambda name: f"/usr/bin/{name}") + + def fail_run(command: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]: + return subprocess.CompletedProcess(command, 2, stdout="", stderr="native failed") + + monkeypatch.setattr("codebase_graph.setup.installer.subprocess.run", fail_run) + + result = install_mcp_server(McpInstallOptions(client="codex", setup_config_path=config_path)) + + assert result.action == "created" + assert result.method == "file_adapter" + assert result.native_command[:4] == ["codex", "mcp", "add", "codebaseGraph-fresh_repo"] + assert result.native_error == "exit 2: native failed" + assert (codex_home / "config.toml").exists() + + +def test_dry_run_never_writes_files_or_calls_native_cli( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config_path = _write_setup_config(tmp_path / "fresh_repo") + monkeypatch.setenv("HOME", tmp_path.as_posix()) + + def fail_run(*args: object, **kwargs: object) -> subprocess.CompletedProcess[str]: + raise AssertionError("dry-run must not call subprocess.run") + + monkeypatch.setattr("codebase_graph.setup.installer.subprocess.run", fail_run) + + result = install_mcp_server( + McpInstallOptions(client="generic", setup_config_path=config_path, dry_run=True) + ) + + assert result.action == "dry_run" + assert result.method == "file_adapter" + assert not (tmp_path / ".config" / "mcp" / "mcp.json").exists() + + +def test_setup_compatibility_uses_legacy_server_name_and_file_adapter( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + pytest.importorskip("tree_sitter") + pytest.importorskip("tree_sitter_python") + pytest.importorskip("real_ladybug") + repo_root = _fresh_repo(tmp_path) + mcp_config_path = tmp_path / "codex.toml" + monkeypatch.setattr("codebase_graph.setup.installer.shutil.which", lambda name: f"/usr/bin/{name}") + + exit_code = cli_main( + [ + "setup", + "--repo-root", + repo_root.as_posix(), + "--mcp-client", + "codex", + "--mcp-config-path", + mcp_config_path.as_posix(), + "--instructions-target", + "skip", + ] + ) + output = json.loads(capsys.readouterr().out) + + assert exit_code == 0 + assert output["mcp_config"]["server_name"] == "codebaseGraph" + assert output["mcp_config"]["method"] == "file_adapter" + assert output["mcp_config"]["path"] == mcp_config_path.as_posix() + + +def test_hermes_default_path_is_documented_home_config( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("HOME", tmp_path.as_posix()) + descriptor = build_server_descriptor(tmp_path / ".codebaseGraph" / "config.json") + + assert get_client_adapter("hermes").default_config_path(descriptor) == tmp_path / ".hermes" / "config.yaml" + + +def test_all_client_install_reports_partial_failure( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + import codebase_graph.setup.installer as installer + + config_path = _write_setup_config(tmp_path / "fresh_repo") + monkeypatch.setenv("CODEX_HOME", (tmp_path / "codex-home").as_posix()) + monkeypatch.setattr(installer, "INSTALL_CLIENTS", ("codex", "generic")) + monkeypatch.setattr(installer.shutil, "which", lambda name: None) + original_get_adapter = installer.get_client_adapter + + def get_adapter(client: str) -> object: + if client == "generic": + raise ValueError("adapter unavailable") + return original_get_adapter(client) + + monkeypatch.setattr(installer, "get_client_adapter", get_adapter) + + results = install_mcp_clients(McpInstallOptions(client="all", setup_config_path=config_path)) + + assert [result.client for result in results] == ["codex", "generic"] + assert results[0].action == "created" + assert results[1].action == "failed" + assert results[1].error == "adapter unavailable" + + +def test_mcp_install_cli_dry_run_json( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + config_path = _write_setup_config(tmp_path / "fresh_repo") + monkeypatch.setattr("codebase_graph.setup.installer.shutil.which", lambda name: f"/usr/bin/{name}") + + exit_code = cli_main( + ["mcp", "install", "--client", "codex", "--config-path", config_path.as_posix(), "--dry-run", "--json"] + ) + output = json.loads(capsys.readouterr().out) + + assert exit_code == 0 + assert output["action"] == "dry_run" + assert output["method"] == "native_cli" + assert output["server_name"] == default_server_name("fresh_repo") + + +def _write_setup_config(repo_root: Path) -> Path: + repo_root.mkdir(parents=True) + paths = derive_setup_paths(repo_root) + mcp_command = ["codebase-graph", "mcp", "serve", "--config", paths.config_path.as_posix()] + payload = build_setup_config(paths, mcp_command=mcp_command) + write_setup_config(paths.config_path, payload) + return paths.config_path + + +def _fresh_repo(tmp_path: Path) -> Path: + repo_root = tmp_path / "fresh_repo" + package = repo_root / "sample_project" + package.mkdir(parents=True) + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "service.py").write_text( + "class SampleService:\n" + " def run(self) -> str:\n" + " return helper()\n\n" + "def helper() -> str:\n" + " return 'ok'\n", + encoding="utf-8", + ) + return repo_root From 6c12cd0220f75af614b85f22c00800a584fa1393 Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Mon, 25 May 2026 15:13:04 +0930 Subject: [PATCH 16/53] fix: mcp tool exposure --- .github/workflows/ci.yml | 108 +++++++++++++++++++++++ .github/workflows/release.yml | 120 ++++++++++++++++++++++++++ AGENTS.md | 6 +- README.md | 18 ++-- conda-forge/recipe/meta.yaml | 52 +++++++++++ docs/release.md | 48 +++++++++++ pyproject.toml | 10 ++- src/codebase_graph/cli/__init__.py | 2 +- src/codebase_graph/mcp/protocol.py | 4 +- src/codebase_graph/paths.py | 2 +- src/codebase_graph/setup/__init__.py | 25 +++++- src/codebase_graph/setup/installer.py | 4 +- tests/test_mcp_entrypoint.py | 21 +++++ tests/test_mcp_installer.py | 20 +++-- tests/test_mcp_portability.py | 16 ++-- tests/test_setup_workflow.py | 6 +- 16 files changed, 425 insertions(+), 37 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/release.yml create mode 100644 conda-forge/recipe/meta.yaml create mode 100644 docs/release.md create mode 100644 tests/test_mcp_entrypoint.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..adf2d0a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,108 @@ +name: CI + +on: + pull_request: + branches: + - main + - "codex/**" + push: + branches: + - main + - "codex/**" + +permissions: + contents: read + +jobs: + test: + name: pytest (${{ matrix.os }}, py${{ matrix.python-version }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - ubuntu-latest + - macos-latest + - windows-latest + python-version: + - "3.10" + - "3.11" + - "3.12" + - "3.13" + - "3.14" + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: pip + + - name: Install package and test dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -e ".[dev]" + + - name: Run tests + run: python -m pytest -q + + lint: + name: ruff + runs-on: ubuntu-latest + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + + - name: Install lint dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -e ".[dev]" + + - name: Run ruff + run: ruff check . + + package: + name: package + runs-on: ubuntu-latest + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + + - name: Build distributions + run: | + python -m pip install --upgrade pip + python -m pip install build twine + python -m build + python -m twine check dist/* + + - name: Smoke-test built wheel + shell: bash + run: | + python -m venv /tmp/codebase-graph-wheel + /tmp/codebase-graph-wheel/bin/python -m pip install --upgrade pip + /tmp/codebase-graph-wheel/bin/python -m pip install dist/*.whl + /tmp/codebase-graph-wheel/bin/codebase-graph --help + /tmp/codebase-graph-wheel/bin/codebase-graph-mcp --help diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..d2190ef --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,120 @@ +name: Release + +on: + push: + tags: + - "v*.*.*" + workflow_dispatch: + inputs: + tag: + description: "Release tag to publish, in vX.Y.Z format" + required: true + type: string + +permissions: + contents: read + +jobs: + build: + name: build release distributions + runs-on: ubuntu-latest + outputs: + package-version: ${{ steps.verify-version.outputs.package-version }} + + env: + RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref_name }} + + steps: + - name: Check out release tag + uses: actions/checkout@v4 + with: + ref: ${{ env.RELEASE_TAG }} + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + + - name: Build and validate distributions + run: | + python -m pip install --upgrade pip + python -m pip install build twine + python -m build + python -m twine check dist/* + + - name: Verify package version matches tag + id: verify-version + shell: bash + run: | + python - <<'PY' + import email + import glob + import os + import re + import tarfile + import zipfile + + tag = os.environ["RELEASE_TAG"] + match = re.fullmatch(r"v(?P\d+\.\d+\.\d+)", tag) + if match is None: + raise SystemExit(f"release tag must match vX.Y.Z, got {tag!r}") + expected = match.group("version") + + versions = set() + for artifact in glob.glob("dist/*"): + if artifact.endswith(".whl"): + with zipfile.ZipFile(artifact) as wheel: + metadata_name = next(name for name in wheel.namelist() if name.endswith(".dist-info/METADATA")) + metadata = email.message_from_bytes(wheel.read(metadata_name)) + elif artifact.endswith(".tar.gz"): + with tarfile.open(artifact, "r:gz") as sdist: + metadata_member = next(member for member in sdist.getmembers() if member.name.endswith("/PKG-INFO")) + metadata = email.message_from_binary_file(sdist.extractfile(metadata_member)) + else: + raise SystemExit(f"unexpected distribution artifact: {artifact}") + versions.add(metadata["Version"]) + + if versions != {expected}: + raise SystemExit(f"built package versions {sorted(versions)} do not match release tag {tag}") + + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: + output.write(f"package-version={expected}\n") + PY + + - name: Smoke-test built wheel + shell: bash + run: | + python -m venv /tmp/codebase-graph-wheel + /tmp/codebase-graph-wheel/bin/python -m pip install --upgrade pip + /tmp/codebase-graph-wheel/bin/python -m pip install dist/*.whl + /tmp/codebase-graph-wheel/bin/codebase-graph --help + /tmp/codebase-graph-wheel/bin/codebase-graph-mcp --help + + - name: Upload distributions + uses: actions/upload-artifact@v4 + with: + name: codebase-graph-${{ steps.verify-version.outputs.package-version }}-dist + path: dist/* + if-no-files-found: error + + publish-pypi: + name: publish to PyPI + needs: build + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/codebase-graph + permissions: + id-token: write + + steps: + - name: Download distributions + uses: actions/download-artifact@v4 + with: + name: codebase-graph-${{ needs.build.outputs.package-version }}-dist + path: dist + + - name: Publish distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/AGENTS.md b/AGENTS.md index d07a77a..3df3bb1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,8 +1,8 @@ ## codebaseGraph workflow -- Treat the `codebaseGraph` MCP server knowledge graph as the project operating source of truth. -- Use the `codebaseGraph` MCP server for repository graph search, schema, and compact context before answering repo-structure questions or performing coding tasks. +- Treat the `codebase_graph` MCP server knowledge graph as the project operating source of truth. +- Use the `codebase_graph` MCP server for repository graph search, schema, and compact context before answering repo-structure questions or performing coding tasks. - Prefer `graph_search` for symbols, paths, docs, and setup instructions; follow with `graph_context` when relationships or nearby evidence matter. - Use `graph_schema` or `graph_query_helpers` before writing raw graph queries, and keep `graph_query` read-only. -- Refresh the graph with `/Users/rabii/Projects/Repositories/codebaseGraph/.venv/bin/codebase-graph setup --repo-root . --mcp-client codex` when files change materially. Setup config: `/Users/rabii/Projects/Repositories/codebaseGraph/.codebaseGraph/config.json`. +- Refresh the graph with `/Users/rabii/Projects/Repositories/codebaseGraph/.venv/bin/codebase-graph setup --repo-root . --mcp-client none` when files change materially; install or update MCP with `/Users/rabii/Projects/Repositories/codebaseGraph/.venv/bin/codebase-graph mcp install --client codex`. Setup config: `/Users/rabii/Projects/Repositories/codebaseGraph/.codebaseGraph/config.json`. diff --git a/README.md b/README.md index ca4b0c8..704340a 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ The setup command also: - Materializes the repository graph into the repo-local database. - Writes or updates one marked codebaseGraph block in `AGENTS.md` or `CLAUDE.md`. -- Installs an MCP client entry named `codebaseGraph`, unless skipped. +- Installs an MCP client entry named `codebase_graph`, unless skipped. Useful options: @@ -60,7 +60,7 @@ The user-facing installer is: codebase-graph mcp install ``` -By default this installs Codex with a repository-specific server name, for example `codebaseGraph-my-service`. It builds the server descriptor from `.codebaseGraph/config.json`, uses the supported native client CLI when available, and falls back to the adapter file writer when the CLI is missing or fails. +By default this installs Codex with a repository-specific server name, for example `codebase_graph_my_service`. It builds the server descriptor from `.codebaseGraph/config.json`, uses the supported native client CLI when available, and falls back to the adapter file writer when the CLI is missing or fails. Useful installer options: @@ -73,7 +73,7 @@ codebase-graph mcp install --client hermes codebase-graph mcp install --client openclaw codebase-graph mcp install --client generic codebase-graph mcp install --client all --dry-run --json -codebase-graph mcp install --name codebaseGraph +codebase-graph mcp install --name codebase_graph codebase-graph mcp install --config-path /path/to/.codebaseGraph/config.json codebase-graph mcp install --verify ``` @@ -86,7 +86,7 @@ claude mcp add --transport stdio --scope -- openclaw mcp set '' ``` -If native installation is unavailable, codebaseGraph writes the client config file directly. `setup --mcp-client ...` remains supported and delegates to the same installer behavior after materializing graph state and updating instructions. For backward compatibility, setup still uses the legacy fixed server name `codebaseGraph`; use `codebase-graph mcp install --name codebaseGraph` if you want that name from the installer too. +If native installation is unavailable, codebaseGraph writes the client config file directly. `setup --mcp-client ...` remains supported and delegates to the same installer behavior after materializing graph state and updating instructions. The default MCP server name is `codebase_graph`, which avoids mixed-case tool namespace issues in clients that normalize or validate MCP labels strictly. `--dry-run` reports the native command or emitted file patch without calling native CLIs or writing files. `--verify` runs a direct stdio MCP smoke test and, where available, asks the client CLI whether it can see the server. @@ -97,7 +97,7 @@ Setup and install build one canonical server descriptor and serialize it into th Codex uses `~/.codex/config.toml`: ```toml -[mcp_servers.codebaseGraph] +[mcp_servers.codebase_graph] command = "codebase-graph" args = ["mcp", "serve", "--config", ".codebaseGraph/config.json"] startup_timeout_sec = 60 @@ -108,7 +108,7 @@ Claude Desktop, Claude project config, LM Studio, and generic MCP JSON use an `m ```json { "mcpServers": { - "codebaseGraph": { + "codebase_graph": { "type": "stdio", "command": "codebase-graph", "args": ["mcp", "serve", "--config", ".codebaseGraph/config.json"] @@ -179,6 +179,12 @@ python -m pytest ruff check . ``` +## CI and releases + +GitHub Actions runs pytest across Linux, macOS, and Windows for Python 3.10 through 3.14, plus ruff and package-build validation. Releases are driven by `vX.Y.Z` tags, use tag-derived package versions, and publish to PyPI through Trusted Publishing. + +Conda distribution uses the conda-forge staged-recipes path rather than direct Anaconda.org uploads. See [docs/release.md](docs/release.md) for the release workflow and conda-forge submission checklist. + ## Troubleshooting - Missing LadyBugDB: install a package build that includes `real_ladybug`; setup will fail before creating `.codebaseGraph`. diff --git a/conda-forge/recipe/meta.yaml b/conda-forge/recipe/meta.yaml new file mode 100644 index 0000000..ed09224 --- /dev/null +++ b/conda-forge/recipe/meta.yaml @@ -0,0 +1,52 @@ +{% set name = "codebase-graph" %} +{% set pypi_name = "codebase_graph" %} +{% set version = "0.1.0" %} +{% set python_min = "3.10" %} + +package: + name: {{ name|lower }} + version: {{ version }} + +source: + url: https://pypi.org/packages/source/{{ name[0] }}/{{ name }}/{{ pypi_name }}-{{ version }}.tar.gz + sha256: PUT_RELEASE_SDIST_SHA256_HERE + +build: + noarch: python + number: 0 + script: {{ PYTHON }} -m pip install . -vv --no-deps --no-build-isolation + entry_points: + - codebase-graph = codebase_graph.cli:main + - codebase-graph-mcp = codebase_graph.mcp.server:main + +requirements: + host: + - python {{ python_min }} + - pip + - setuptools >=68 + - setuptools-scm >=8 + - wheel + run: + - python >={{ python_min }} + - real-ladybug + - tomli # [py<311] + - tree-sitter + - tree-sitter-python + +test: + imports: + - codebase_graph + commands: + - codebase-graph --help + - codebase-graph-mcp --help + requires: + - python {{ python_min }} + +about: + home: https://github.com/rabii-chaarani/codebaseGraph + summary: Generic codebase knowledge graph engine for Python projects. + license: PUT_SPDX_LICENSE_ID_HERE + +extra: + recipe-maintainers: + - rabii-chaarani diff --git a/docs/release.md b/docs/release.md new file mode 100644 index 0000000..043aeba --- /dev/null +++ b/docs/release.md @@ -0,0 +1,48 @@ +# Release Process + +`codebaseGraph` releases are tag-driven. The GitHub release workflow builds the source distribution and wheel from a `vX.Y.Z` tag, verifies that the package metadata version matches the tag, and publishes to PyPI with Trusted Publishing. + +## One-time PyPI setup + +Configure a PyPI Trusted Publisher for: + +- PyPI project: `codebase-graph` +- Owner/repository: `rabii-chaarani/codebaseGraph` +- Workflow: `release.yml` +- Environment: `pypi` + +Create the `pypi` GitHub environment before the first release. Use required reviewers on that environment when release approval should be manual. + +## CI + +Pull requests and pushes to `main` or `codex/**` run: + +- `pytest` on Linux, macOS, and Windows for Python 3.10 through 3.14. +- `ruff check .` on Linux. +- A package build on Linux with `python -m build`, `twine check`, and console-script smoke tests from the built wheel. + +## PyPI release + +1. Confirm the release branch is green in CI. +2. Create an intentional release tag: + + ```bash + git tag vX.Y.Z + git push origin vX.Y.Z + ``` + +3. The `Release` workflow checks out that tag, builds the distributions, verifies `Version: X.Y.Z`, and publishes to PyPI from the protected `pypi` environment. + +For manual reruns, use the `Release` workflow dispatch input with an existing `vX.Y.Z` tag. + +## Conda-forge release path + +This repository intentionally does not upload directly to Anaconda.org. Conda distribution should go through conda-forge: + +1. Ensure the PyPI release has completed and download the source distribution SHA256. +2. Before submitting `codebase-graph`, verify all runtime dependencies exist on conda-forge. If `real-ladybug` is not available, package that dependency first. +3. Copy `conda-forge/recipe/meta.yaml` into a new `recipes/codebase-graph/` directory in a fork of `conda-forge/staged-recipes`. +4. Replace `version`, `sha256`, and `license` placeholders with release-specific values. +5. Open the staged-recipes pull request and let conda-forge CI validate Linux, macOS, and Windows builds. + +After staged-recipes is merged, future conda releases are handled in the generated `codebase-graph-feedstock`. diff --git a/pyproject.toml b/pyproject.toml index 6b1c58f..258ac10 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,10 @@ [build-system] -requires = ["setuptools>=68", "wheel"] +requires = ["setuptools>=68", "setuptools-scm>=8", "wheel"] build-backend = "setuptools.build_meta" [project] name = "codebase-graph" -version = "0.1.0" +dynamic = ["version"] description = "Generic codebase knowledge graph engine for Python projects." readme = "README.md" requires-python = ">=3.10" @@ -20,6 +20,8 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Topic :: Software Development :: Libraries :: Python Modules", ] @@ -37,6 +39,10 @@ where = ["src"] include = ["codebase_graph*"] exclude = ["tests*", "codebase_graph.egg-info*"] +[tool.setuptools_scm] +version_scheme = "guess-next-dev" +local_scheme = "no-local-version" + [tool.ruff] line-length = 120 target-version = "py310" diff --git a/src/codebase_graph/cli/__init__.py b/src/codebase_graph/cli/__init__.py index 12ef09e..c272372 100644 --- a/src/codebase_graph/cli/__init__.py +++ b/src/codebase_graph/cli/__init__.py @@ -51,7 +51,7 @@ def main(argv: Sequence[str] | None = None) -> int: install_parser = mcp_subparsers.add_parser("install", help="Install the MCP server in a supported client") install_parser.add_argument("--client", choices=supported_install_client_ids(include_all=True), default="codex") install_parser.add_argument("--scope", choices=("local", "user", "project"), default="local") - install_parser.add_argument("--name", default=None, help="MCP server name; defaults to codebaseGraph-") + install_parser.add_argument("--name", default=None, help="MCP server name; defaults to codebase_graph-") install_parser.add_argument("--config-path", default=None, help="Path to .codebaseGraph/config.json") install_parser.add_argument("--repo-root", default=".", help="Repository root used to find .codebaseGraph/config.json") install_parser.add_argument("--dry-run", action="store_true", help="Show the install action without writing or invoking CLIs") diff --git a/src/codebase_graph/mcp/protocol.py b/src/codebase_graph/mcp/protocol.py index 8990a1f..c466b50 100644 --- a/src/codebase_graph/mcp/protocol.py +++ b/src/codebase_graph/mcp/protocol.py @@ -3,6 +3,8 @@ from dataclasses import dataclass from typing import Any +from codebase_graph.paths import MCP_SERVER_NAME + from .runtime import GraphRuntimeConfig, package_version from .tools import UnknownToolError, call_tool_result, tool_specs @@ -74,7 +76,7 @@ def _initialize(self, params: dict[str, Any]) -> dict[str, Any]: return { "protocolVersion": protocol_version, "capabilities": {"tools": {"listChanged": False}}, - "serverInfo": {"name": "codebaseGraph", "version": package_version()}, + "serverInfo": {"name": MCP_SERVER_NAME, "version": package_version()}, } def _call_tool(self, params: dict[str, Any]) -> dict[str, Any]: diff --git a/src/codebase_graph/paths.py b/src/codebase_graph/paths.py index ed373aa..f89fe2b 100644 --- a/src/codebase_graph/paths.py +++ b/src/codebase_graph/paths.py @@ -6,7 +6,7 @@ DEFAULT_STATE_DIR = ".codebaseGraph" CONFIG_NAME = "config.json" MANIFEST_NAME = "manifest.json" -MCP_SERVER_NAME = "codebaseGraph" +MCP_SERVER_NAME = "codebase_graph" @dataclass(frozen=True, slots=True) diff --git a/src/codebase_graph/setup/__init__.py b/src/codebase_graph/setup/__init__.py index 8abc481..eba8a55 100644 --- a/src/codebase_graph/setup/__init__.py +++ b/src/codebase_graph/setup/__init__.py @@ -1,7 +1,8 @@ """Production setup orchestration for repository graph bootstrapping.""" -from .installer import McpInstallOptions, McpInstallResult, install_mcp_clients, install_mcp_server -from .orchestrator import SetupError, SetupOptions, SetupResult, run_setup +from importlib import import_module +from typing import Any + from .state import ( CONFIG_NAME, DEFAULT_STATE_DIR, @@ -13,6 +14,17 @@ load_setup_config, ) +_LAZY_EXPORTS = { + "McpInstallOptions": (".installer", "McpInstallOptions"), + "McpInstallResult": (".installer", "McpInstallResult"), + "SetupError": (".orchestrator", "SetupError"), + "SetupOptions": (".orchestrator", "SetupOptions"), + "SetupResult": (".orchestrator", "SetupResult"), + "install_mcp_clients": (".installer", "install_mcp_clients"), + "install_mcp_server": (".installer", "install_mcp_server"), + "run_setup": (".orchestrator", "run_setup"), +} + __all__ = [ "CONFIG_NAME", "DEFAULT_STATE_DIR", @@ -31,3 +43,12 @@ "install_mcp_server", "run_setup", ] + + +def __getattr__(name: str) -> Any: + if name not in _LAZY_EXPORTS: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + module_name, attribute_name = _LAZY_EXPORTS[name] + value = getattr(import_module(module_name, __name__), attribute_name) + globals()[name] = value + return value diff --git a/src/codebase_graph/setup/installer.py b/src/codebase_graph/setup/installer.py index 82b5696..2799b94 100644 --- a/src/codebase_graph/setup/installer.py +++ b/src/codebase_graph/setup/installer.py @@ -93,7 +93,7 @@ def supported_install_client_ids(*, include_all: bool = False) -> tuple[str, ... def default_server_name(repo_name: str | None) -> str: safe_repo_name = _safe_name(repo_name or "repository") - return f"{MCP_SERVER_NAME}-{safe_repo_name}" + return f"{MCP_SERVER_NAME}_{safe_repo_name}" def install_mcp_clients(options: McpInstallOptions) -> list[McpInstallResult]: @@ -508,4 +508,4 @@ def _subprocess_error(completed: subprocess.CompletedProcess[str]) -> str: def _safe_name(value: str) -> str: normalized = re.sub(r"[^A-Za-z0-9_-]+", "_", value.strip()) - return normalized.strip("._-") or "repository" + return normalized.strip("._-").lower() or "repository" diff --git a/tests/test_mcp_entrypoint.py b/tests/test_mcp_entrypoint.py new file mode 100644 index 0000000..dbc876b --- /dev/null +++ b/tests/test_mcp_entrypoint.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +import subprocess +import sys + + +def test_mcp_entrypoint_help_imports_without_setup_cycle() -> None: + completed = subprocess.run( + [ + sys.executable, + "-c", + "from codebase_graph.mcp.server import main; raise SystemExit(main())", + "--help", + ], + capture_output=True, + text=True, + check=False, + ) + + assert completed.returncode == 0, completed.stderr + assert "usage: codebase-graph-mcp" in completed.stdout diff --git a/tests/test_mcp_installer.py b/tests/test_mcp_installer.py index 8f4353b..5aa872f 100644 --- a/tests/test_mcp_installer.py +++ b/tests/test_mcp_installer.py @@ -18,6 +18,10 @@ from codebase_graph.setup.state import build_setup_config, derive_setup_paths, write_setup_config +def test_default_server_name_is_namespace_safe() -> None: + assert default_server_name("My Service") == "codebase_graph_my_service" + + def test_codex_native_command_generation_uses_repo_server_name( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -29,8 +33,8 @@ def test_codex_native_command_generation_uses_repo_server_name( assert result.action == "dry_run" assert result.method == "native_cli" - assert result.server_name == "codebaseGraph-fresh_repo" - assert result.command[:4] == ["codex", "mcp", "add", "codebaseGraph-fresh_repo"] + assert result.server_name == "codebase_graph_fresh_repo" + assert result.command[:4] == ["codex", "mcp", "add", "codebase_graph_fresh_repo"] assert result.command[4] == "--" @@ -53,7 +57,7 @@ def test_claude_native_command_includes_transport_and_scope( "stdio", "--scope", "user", - "codebaseGraph-fresh_repo", + "codebase_graph_fresh_repo", ] @@ -68,7 +72,7 @@ def test_claude_project_native_command_forces_project_scope( McpInstallOptions(client="claude-project", scope="user", setup_config_path=config_path, dry_run=True) ) - assert result.command[6:8] == ["project", "codebaseGraph-fresh_repo"] + assert result.command[6:8] == ["project", "codebase_graph_fresh_repo"] def test_openclaw_native_command_emits_server_json( @@ -83,7 +87,7 @@ def test_openclaw_native_command_emits_server_json( ) entry = json.loads(result.command[-1]) - assert result.command[:4] == ["openclaw", "mcp", "set", "codebaseGraph-fresh_repo"] + assert result.command[:4] == ["openclaw", "mcp", "set", "codebase_graph_fresh_repo"] assert entry["type"] == "stdio" assert entry["args"][:2] == ["mcp", "serve"] @@ -124,7 +128,7 @@ def fail_run(command: list[str], **kwargs: object) -> subprocess.CompletedProces assert result.action == "created" assert result.method == "file_adapter" - assert result.native_command[:4] == ["codex", "mcp", "add", "codebaseGraph-fresh_repo"] + assert result.native_command[:4] == ["codex", "mcp", "add", "codebase_graph_fresh_repo"] assert result.native_error == "exit 2: native failed" assert (codex_home / "config.toml").exists() @@ -150,7 +154,7 @@ def fail_run(*args: object, **kwargs: object) -> subprocess.CompletedProcess[str assert not (tmp_path / ".config" / "mcp" / "mcp.json").exists() -def test_setup_compatibility_uses_legacy_server_name_and_file_adapter( +def test_setup_compatibility_uses_snake_case_server_name_and_file_adapter( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], @@ -178,7 +182,7 @@ def test_setup_compatibility_uses_legacy_server_name_and_file_adapter( output = json.loads(capsys.readouterr().out) assert exit_code == 0 - assert output["mcp_config"]["server_name"] == "codebaseGraph" + assert output["mcp_config"]["server_name"] == "codebase_graph" assert output["mcp_config"]["method"] == "file_adapter" assert output["mcp_config"]["path"] == mcp_config_path.as_posix() diff --git a/tests/test_mcp_portability.py b/tests/test_mcp_portability.py index 2000a5b..c6967e6 100644 --- a/tests/test_mcp_portability.py +++ b/tests/test_mcp_portability.py @@ -79,14 +79,14 @@ def test_client_adapters_emit_native_config_shapes(tmp_path: Path) -> None: codex_patch = rendered["codex"]["patch"] codex_payload = tomllib.loads(codex_patch) - assert codex_payload["mcp_servers"]["codebaseGraph"]["command"] - assert codex_payload["mcp_servers"]["codebaseGraph"]["startup_timeout_sec"] == 60 - assert "type" not in rendered["claude"]["payload"]["mcpServers"]["codebaseGraph"] - assert rendered["claude"]["payload"]["mcpServers"]["codebaseGraph"]["command"] - assert rendered["claude-project"]["payload"]["mcpServers"]["codebaseGraph"]["type"] == "stdio" - assert rendered["lmstudio"]["payload"]["mcpServers"]["codebaseGraph"]["type"] == "stdio" - assert rendered["generic"]["payload"]["mcpServers"]["codebaseGraph"]["args"][0:2] == ["mcp", "serve"] - assert rendered["openclaw"]["payload"]["mcp"]["servers"]["codebaseGraph"]["type"] == "stdio" + assert codex_payload["mcp_servers"]["codebase_graph"]["command"] + assert codex_payload["mcp_servers"]["codebase_graph"]["startup_timeout_sec"] == 60 + assert "type" not in rendered["claude"]["payload"]["mcpServers"]["codebase_graph"] + assert rendered["claude"]["payload"]["mcpServers"]["codebase_graph"]["command"] + assert rendered["claude-project"]["payload"]["mcpServers"]["codebase_graph"]["type"] == "stdio" + assert rendered["lmstudio"]["payload"]["mcpServers"]["codebase_graph"]["type"] == "stdio" + assert rendered["generic"]["payload"]["mcpServers"]["codebase_graph"]["args"][0:2] == ["mcp", "serve"] + assert rendered["openclaw"]["payload"]["mcp"]["servers"]["codebase_graph"]["type"] == "stdio" assert "mcp_servers:" in rendered["hermes"]["patch"] diff --git a/tests/test_setup_workflow.py b/tests/test_setup_workflow.py index 3ef79b9..2558d11 100644 --- a/tests/test_setup_workflow.py +++ b/tests/test_setup_workflow.py @@ -56,7 +56,7 @@ def test_setup_cli_creates_state_db_mcp_config_instructions_and_searchable_docs( assert agents_text.count(END_MARKER) == 1 mcp_payload = tomllib.loads(mcp_config_path.read_text(encoding="utf-8")) assert "otherServer" not in mcp_payload.get("mcp_servers", {}) - assert mcp_payload["mcp_servers"]["codebaseGraph"]["args"] == [ + assert mcp_payload["mcp_servers"]["codebase_graph"]["args"] == [ "mcp", "serve", "--config", @@ -112,7 +112,7 @@ def test_mcp_config_dry_run_preserves_existing_json_servers(tmp_path: Path) -> N ) assert dry_run.action == "dry_run" - assert "codebaseGraph" not in json.loads(config_path.read_text(encoding="utf-8"))["mcpServers"] + assert "codebase_graph" not in json.loads(config_path.read_text(encoding="utf-8"))["mcpServers"] written = configure_mcp_client( client="generic", @@ -122,7 +122,7 @@ def test_mcp_config_dry_run_preserves_existing_json_servers(tmp_path: Path) -> N payload = json.loads(config_path.read_text(encoding="utf-8")) assert written.action == "created" - assert set(payload["mcpServers"]) == {"otherServer", "codebaseGraph"} + assert set(payload["mcpServers"]) == {"otherServer", "codebase_graph"} def test_server_entry_prefers_current_environment_script(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: From 1cbec0f8952465297f0afa31403655af8020047a Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Mon, 25 May 2026 16:06:03 +0930 Subject: [PATCH 17/53] Restore CI coverage across supported Python and Windows runners The latest GitHub Actions CI run failed on Python 3.10 because materializer imported datetime.UTC, which is only available on Python 3.11+, while the package declares support for Python 3.10. Windows pytest jobs also failed because the Hermes default-path test patched HOME even though pathlib resolves the platform profile path on Windows. This keeps runtime behavior unchanged by using timezone.utc for aware timestamps and makes the test patch the home resolver directly instead of relying on platform-specific environment handling. Constraint: pyproject.toml advertises Python >=3.10 and the CI matrix runs Windows, macOS, and Ubuntu. Rejected: Drop Python 3.10 from the CI matrix | advertised package support still includes Python 3.10. Rejected: Change Hermes production path resolution to read HOME first | Path.home() is the production behavior; the test fixture was the portability problem. Confidence: high Scope-risk: narrow Reversibility: clean Directive: Do not reintroduce stdlib APIs newer than the declared minimum Python without raising requires-python and CI together. Tested: ./.venv/bin/python -m pytest -q; ./.venv/bin/ruff check .; git diff --check; ./.venv/bin/codebase-graph setup --repo-root . --mcp-client none Not-tested: Local Python 3.10 runtime unavailable; GitHub Actions rerun pending --- src/codebase_graph/ingest/materializer.py | 6 +++--- tests/test_mcp_installer.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/codebase_graph/ingest/materializer.py b/src/codebase_graph/ingest/materializer.py index b6b7e9f..f49946c 100644 --- a/src/codebase_graph/ingest/materializer.py +++ b/src/codebase_graph/ingest/materializer.py @@ -6,7 +6,7 @@ import tempfile from collections.abc import Mapping from dataclasses import dataclass, field -from datetime import UTC, datetime +from datetime import datetime, timezone from pathlib import Path from typing import Any, Literal @@ -505,7 +505,7 @@ def _write_rebuild_marker(marker_path: Path, db_path: Path, manifest_path: Path) with tmp_path.open("w", encoding="utf-8") as handle: json.dump( { - "created_at": datetime.now(UTC).isoformat(), + "created_at": datetime.now(timezone.utc).isoformat(), "db_path": db_path.as_posix(), "manifest_path": manifest_path.as_posix(), }, @@ -560,7 +560,7 @@ def _manifest_entry(snapshot: SourceSnapshot, graph: CodeGraph) -> ManifestEntry edge_ids=tuple(sorted(graph.edges)), node_types={node_id: node.table for node_id, node in graph.nodes.items()}, edge_types={edge_id: edge.type for edge_id, edge in graph.edges.items()}, - materialized_at=datetime.now(UTC).isoformat(), + materialized_at=datetime.now(timezone.utc).isoformat(), ) diff --git a/tests/test_mcp_installer.py b/tests/test_mcp_installer.py index 5aa872f..ca2f3f1 100644 --- a/tests/test_mcp_installer.py +++ b/tests/test_mcp_installer.py @@ -191,7 +191,7 @@ def test_hermes_default_path_is_documented_home_config( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.setenv("HOME", tmp_path.as_posix()) + monkeypatch.setattr(Path, "home", classmethod(lambda cls: tmp_path)) descriptor = build_server_descriptor(tmp_path / ".codebaseGraph" / "config.json") assert get_client_adapter("hermes").default_config_path(descriptor) == tmp_path / ".hermes" / "config.yaml" From b29a01bb7a55e70e3f3aeddf813c2129144e1400 Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Wed, 27 May 2026 10:16:16 +0930 Subject: [PATCH 18/53] feat: add architecture query catalog --- AGENTS.md | 1 + README.md | 3 + src/codebase_graph/mcp/tools.py | 24 +- src/codebase_graph/reasoning/__init__.py | 17 +- .../reasoning/architecture_queries.py | 372 ++++++++++++++++++ src/codebase_graph/setup/instructions.py | 1 + tests/test_architecture_queries.py | 93 +++++ tests/test_mcp_portability.py | 43 ++ tests/test_setup_workflow.py | 1 + 9 files changed, 553 insertions(+), 2 deletions(-) create mode 100644 src/codebase_graph/reasoning/architecture_queries.py create mode 100644 tests/test_architecture_queries.py diff --git a/AGENTS.md b/AGENTS.md index 3df3bb1..d336baf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,6 +3,7 @@ - Treat the `codebase_graph` MCP server knowledge graph as the project operating source of truth. - Use the `codebase_graph` MCP server for repository graph search, schema, and compact context before answering repo-structure questions or performing coding tasks. - Prefer `graph_search` for symbols, paths, docs, and setup instructions; follow with `graph_context` when relationships or nearby evidence matter. +- For coding tasks that requires architecture orientation, call `graph_architecture_queries` first, then execute selected statements through `graph_query` that relevant to your task. - Use `graph_schema` or `graph_query_helpers` before writing raw graph queries, and keep `graph_query` read-only. - Refresh the graph with `/Users/rabii/Projects/Repositories/codebaseGraph/.venv/bin/codebase-graph setup --repo-root . --mcp-client none` when files change materially; install or update MCP with `/Users/rabii/Projects/Repositories/codebaseGraph/.venv/bin/codebase-graph mcp install --client codex`. Setup config: `/Users/rabii/Projects/Repositories/codebaseGraph/.codebaseGraph/config.json`. diff --git a/README.md b/README.md index 704340a..1cb0e66 100644 --- a/README.md +++ b/README.md @@ -153,8 +153,11 @@ Available MCP tools: - `graph_context` - `graph_schema` - `graph_query_helpers` +- `graph_architecture_queries` - `graph_query` with write-like statements blocked +For coding-task architecture orientation, call `graph_architecture_queries` first to fetch the grouped read-only Cypher catalog, then run selected statements with `graph_query`. + ## CLI search The legacy materializer/search commands are still available. Setup reports the explicit database and manifest paths to use with them: diff --git a/src/codebase_graph/mcp/tools.py b/src/codebase_graph/mcp/tools.py index f6bd39e..89f1239 100644 --- a/src/codebase_graph/mcp/tools.py +++ b/src/codebase_graph/mcp/tools.py @@ -6,7 +6,7 @@ from codebase_graph.db import LadybugCodeGraphStore from codebase_graph.ontology import QUERY_HELPERS, schema_payload -from codebase_graph.reasoning import CompactContextBuilder +from codebase_graph.reasoning import CompactContextBuilder, architecture_query_catalog from codebase_graph.retrieval import SearchRequest, SearchService from .runtime import GraphRuntimeConfig, open_graph_store @@ -28,6 +28,8 @@ def handle_tool_call(name: str, arguments: dict[str, Any], *, runtime: GraphRunt return schema_payload() if name == "graph_query_helpers": return {"query_helpers": [helper.as_dict() for helper in QUERY_HELPERS]} + if name == "graph_architecture_queries": + return architecture_query_catalog(group=_optional_str(arguments.get("group"))) if name == "graph_search": with open_graph_store(runtime) as store: request = _search_request(arguments) @@ -101,6 +103,20 @@ def tool_specs() -> list[dict[str, Any]]: "description": "Return named read-only query helpers for common graph exploration tasks.", "inputSchema": {"type": "object", "properties": {}, "additionalProperties": False}, }, + { + "name": "graph_architecture_queries", + "description": "Return the grouped architecture-discovery Cypher catalog for coding-agent first-step orientation.", + "inputSchema": { + "type": "object", + "properties": { + "group": { + "type": "string", + "description": "Optional architecture query group to return.", + }, + }, + "additionalProperties": False, + }, + }, { "name": "graph_query", "description": "Execute a restricted read-only graph query against the configured database.", @@ -226,3 +242,9 @@ def _optional_int(value: Any) -> int | None: if value is None or value == "": return None return int(value) + + +def _optional_str(value: Any) -> str | None: + if value is None or value == "": + return None + return str(value) diff --git a/src/codebase_graph/reasoning/__init__.py b/src/codebase_graph/reasoning/__init__.py index 44b7855..4f3d7b4 100644 --- a/src/codebase_graph/reasoning/__init__.py +++ b/src/codebase_graph/reasoning/__init__.py @@ -1,5 +1,20 @@ """Path explanation, causal trace, and context assembly.""" +from .architecture_queries import ( + ARCHITECTURE_QUERY_GROUPS, + ARCHITECTURE_QUERY_ORDER, + ArchitectureQueryGroup, + ArchitectureQuerySpec, + architecture_query_catalog, +) from .context_builder import CompactContextBuilder, ContextNode -__all__ = ["CompactContextBuilder", "ContextNode"] +__all__ = [ + "ARCHITECTURE_QUERY_GROUPS", + "ARCHITECTURE_QUERY_ORDER", + "ArchitectureQueryGroup", + "ArchitectureQuerySpec", + "CompactContextBuilder", + "ContextNode", + "architecture_query_catalog", +] diff --git a/src/codebase_graph/reasoning/architecture_queries.py b/src/codebase_graph/reasoning/architecture_queries.py new file mode 100644 index 0000000..fb98970 --- /dev/null +++ b/src/codebase_graph/reasoning/architecture_queries.py @@ -0,0 +1,372 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +WORKFLOW_NAME = "coding_task_architecture_discovery" +EXECUTION_TOOL = "graph_query" + + +@dataclass(frozen=True, slots=True) +class ArchitectureQuerySpec: + name: str + description: str + statement: str + parameters: tuple[str, ...] = () + returns: tuple[str, ...] = () + + def as_dict(self) -> dict[str, Any]: + return { + "name": self.name, + "description": self.description, + "statement": self.statement, + "parameters": list(self.parameters), + "returns": list(self.returns), + } + + +@dataclass(frozen=True, slots=True) +class ArchitectureQueryGroup: + name: str + goal: str + queries: tuple[ArchitectureQuerySpec, ...] + + def as_dict(self) -> dict[str, Any]: + return { + "name": self.name, + "goal": self.goal, + "queries": [query.as_dict() for query in self.queries], + } + + +ARCHITECTURE_QUERY_ORDER = ( + "overview", + "public_surface", + "dependency_topology", + "execution_flow", + "runtime_data_security", + "documentation_context", + "graph_quality_gaps", +) + + +ARCHITECTURE_QUERY_GROUPS: dict[str, ArchitectureQueryGroup] = { + "overview": ArchitectureQueryGroup( + name="overview", + goal="Check graph coverage and establish the indexed codebase shape.", + queries=( + ArchitectureQuerySpec( + name="graph_coverage", + description="Count all materialized graph nodes as a quick coverage check.", + statement="MATCH (n) RETURN count(n) AS total_nodes LIMIT 1", + returns=("total_nodes",), + ), + ArchitectureQuerySpec( + name="source_unit_inventory", + description="List materialized modules with source paths and spans.", + statement=( + "MATCH (m:Module) " + "RETURN m.id, m.label, m.qualified_name, m.path, m.line_start, m.line_end " + "ORDER BY m.path LIMIT 200" + ), + returns=("id", "label", "qualified_name", "path", "line_start", "line_end"), + ), + ArchitectureQuerySpec( + name="package_directory_shape", + description="List source files with path and content metadata.", + statement=( + "MATCH (f:File) " + "RETURN f.path, f.label, f.size_bytes, f.content_hash " + "ORDER BY f.path LIMIT 300" + ), + returns=("path", "label", "size_bytes", "content_hash"), + ), + ), + ), + "public_surface": ArchitectureQueryGroup( + name="public_surface", + goal="Find how the library exposes behavior through modules, definitions, or runtime entrypoints.", + queries=( + ArchitectureQuerySpec( + name="public_surface_candidates", + description="Find exposed module surfaces and fallback definition-level public candidates.", + statement=( + "MATCH (m:Module)-[:FROM_Exposes]->(:Exposes)-[:TO_Exposes]->(surface) " + "RETURN 'exposed' AS surface_source, m.label AS module_label, m.path AS module_path, " + "surface.id AS surface_id, surface.label AS surface_label, " + "surface.qualified_name AS surface_qualified_name, surface.path AS surface_path, " + "surface.line_start AS line_start " + "UNION ALL " + "MATCH (m:Module)-[:FROM_Defines]->(:Defines)-[:TO_Defines]->(surface:Class) " + "RETURN 'defined' AS surface_source, m.label AS module_label, m.path AS module_path, " + "surface.id AS surface_id, surface.label AS surface_label, " + "surface.qualified_name AS surface_qualified_name, surface.path AS surface_path, " + "surface.line_start AS line_start " + "UNION ALL " + "MATCH (m:Module)-[:FROM_Defines]->(:Defines)-[:TO_Defines]->(surface:Function) " + "RETURN 'defined' AS surface_source, m.label AS module_label, m.path AS module_path, " + "surface.id AS surface_id, surface.label AS surface_label, " + "surface.qualified_name AS surface_qualified_name, surface.path AS surface_path, " + "surface.line_start AS line_start " + "UNION ALL " + "MATCH (m:Module)-[:FROM_Defines]->(:Defines)-[:TO_Defines]->(surface:Method) " + "RETURN 'defined' AS surface_source, m.label AS module_label, m.path AS module_path, " + "surface.id AS surface_id, surface.label AS surface_label, " + "surface.qualified_name AS surface_qualified_name, surface.path AS surface_path, " + "surface.line_start AS line_start LIMIT 200" + ), + returns=( + "surface_source", + "module_label", + "module_path", + "surface_id", + "surface_label", + "surface_qualified_name", + "surface_path", + "line_start", + ), + ), + ArchitectureQuerySpec( + name="entrypoint_runtime_surface", + description="Find function-level name/path candidates for runtime or CLI entrypoints.", + statement=( + "MATCH (d:Function) " + "WHERE d.label = 'main' OR d.label = 'cli' OR d.label CONTAINS 'server' OR d.path CONTAINS 'cli' " + "RETURN 'name_candidate' AS entrypoint_kind, d.id AS entrypoint_id, d.label AS entrypoint_label, " + "d.path AS entrypoint_path, d.id AS target_id, d.label AS target_label, " + "d.qualified_name AS target_qualified_name, d.path AS target_path, d.line_start AS line_start " + "LIMIT 100" + ), + returns=( + "entrypoint_kind", + "entrypoint_id", + "entrypoint_label", + "entrypoint_path", + "target_id", + "target_label", + "target_qualified_name", + "target_path", + "line_start", + ), + ), + ), + ), + "dependency_topology": ArchitectureQueryGroup( + name="dependency_topology", + goal="Map internal and external dependencies so agents can infer layers and adapters.", + queries=( + ArchitectureQuerySpec( + name="external_dependency_map", + description="Map import declarations to external dependency nodes.", + statement=( + "MATCH (i:ImportDeclaration)-[:FROM_DependsOn]->(:DependsOn)-[:TO_DependsOn]->(d:Dependency) " + "RETURN i.path, i.label AS import_label, d.label AS dependency " + "ORDER BY d.label, i.path LIMIT 300" + ), + returns=("path", "import_label", "dependency"), + ), + ArchitectureQuerySpec( + name="module_import_coupling", + description="List modules and their import declarations as a coupling inventory.", + statement=( + "MATCH (m:Module)-[:FROM_Imports]->(:Imports)-[:TO_Imports]->(i:ImportDeclaration) " + "RETURN m.label, m.path, i.label, i.line_start " + "ORDER BY m.path, i.line_start LIMIT 300" + ), + returns=("module_label", "module_path", "import_label", "line_start"), + ), + ), + ), + "execution_flow": ArchitectureQueryGroup( + name="execution_flow", + goal="Identify important call paths, orchestration nodes, and central implementation flows.", + queries=( + ArchitectureQuerySpec( + name="high_fan_in_definitions", + description="Find definitions with many resolved incoming references.", + statement=( + "MATCH (ref)-[:FROM_ResolvesTo]->(:ResolvesTo)-[:TO_ResolvesTo]->(target:Class) " + "RETURN target.id, target.label, target.qualified_name, target.path, count(ref) AS inbound_refs " + "UNION ALL " + "MATCH (ref)-[:FROM_ResolvesTo]->(:ResolvesTo)-[:TO_ResolvesTo]->(target:Function) " + "RETURN target.id, target.label, target.qualified_name, target.path, count(ref) AS inbound_refs " + "UNION ALL " + "MATCH (ref)-[:FROM_ResolvesTo]->(:ResolvesTo)-[:TO_ResolvesTo]->(target:Method) " + "RETURN target.id, target.label, target.qualified_name, target.path, count(ref) AS inbound_refs " + "UNION ALL " + "MATCH (ref)-[:FROM_ResolvesTo]->(:ResolvesTo)-[:TO_ResolvesTo]->(target:Module) " + "RETURN target.id, target.label, target.qualified_name, target.path, count(ref) AS inbound_refs " + "ORDER BY inbound_refs DESC LIMIT 50" + ), + returns=("id", "label", "qualified_name", "path", "inbound_refs"), + ), + ArchitectureQuerySpec( + name="high_fan_out_callers", + description="Find functions or methods that call many downstream nodes.", + statement=( + "MATCH (caller:Function)-[:FROM_Calls]->(:Calls)-[:TO_Calls]->(callee) " + "RETURN caller.id, caller.label, caller.qualified_name, caller.path, count(callee) AS outgoing_calls " + "UNION ALL " + "MATCH (caller:Method)-[:FROM_Calls]->(:Calls)-[:TO_Calls]->(callee) " + "RETURN caller.id, caller.label, caller.qualified_name, caller.path, count(callee) AS outgoing_calls " + "ORDER BY outgoing_calls DESC LIMIT 50" + ), + returns=("id", "label", "qualified_name", "path", "outgoing_calls"), + ), + ArchitectureQuerySpec( + name="callable_neighborhood", + description="Inspect direct callees for a named callable.", + statement=( + "MATCH (caller)-[:FROM_Calls]->(:Calls)-[:TO_Calls]->(callee) " + "WHERE caller.label = $name OR caller.qualified_name = $name " + "RETURN caller.id, caller.label, caller.qualified_name, callee.id, " + "callee.label, callee.qualified_name, callee.path LIMIT 100" + ), + parameters=("name",), + returns=( + "caller_id", + "caller_label", + "caller_qualified_name", + "callee_id", + "callee_label", + "callee_qualified_name", + "callee_path", + ), + ), + ), + ), + "runtime_data_security": ArchitectureQueryGroup( + name="runtime_data_security", + goal="Expose data access, query execution, secrets, and configuration-sensitive paths.", + queries=( + ArchitectureQuerySpec( + name="data_query_touchpoints", + description="Find actors that execute or construct query nodes.", + statement=( + "MATCH (actor)-[:FROM_ExecutesQuery]->(:ExecutesQuery)-[:TO_ExecutesQuery]->(q:Query) " + "RETURN actor.id, actor.label, actor.qualified_name, actor.path, q.label, q.path, q.line_start " + "LIMIT 100" + ), + returns=( + "actor_id", + "actor_label", + "actor_qualified_name", + "actor_path", + "query_label", + "query_path", + "query_line_start", + ), + ), + ArchitectureQuerySpec( + name="secret_configuration_touchpoints", + description="Find actors linked to secret or sensitive configuration references.", + statement=( + "MATCH (actor)-[:FROM_UsesSecret]->(:UsesSecret)-[:TO_UsesSecret]->(s:SecretRef) " + "RETURN actor.id, actor.label, actor.qualified_name, actor.path, s.label, s.path, s.line_start " + "LIMIT 100" + ), + returns=( + "actor_id", + "actor_label", + "actor_qualified_name", + "actor_path", + "secret_label", + "secret_path", + "secret_line_start", + ), + ), + ), + ), + "documentation_context": ArchitectureQueryGroup( + name="documentation_context", + goal="Link architecture claims to documentation and parser evidence.", + queries=( + ArchitectureQuerySpec( + name="documentation_to_code_links", + description="Find documentation chunks connected to code nodes.", + statement=( + "MATCH (d:DocumentationChunk)-[:FROM_Documents]->(:Documents)-[:TO_Documents]->(n) " + "RETURN d.id, d.label, d.path, n.id, n.label, n.qualified_name, n.path LIMIT 100" + ), + returns=( + "doc_id", + "doc_label", + "doc_path", + "node_id", + "node_label", + "node_qualified_name", + "node_path", + ), + ), + ArchitectureQuerySpec( + name="evidence_for_symbol", + description="Return evidence nodes for a named symbol or qualified name.", + statement=( + "MATCH (n)-[:FROM_EvidencedBy]->(:EvidencedBy)-[:TO_EvidencedBy]->(e) " + "WHERE n.label = $name OR n.qualified_name = $name " + "RETURN n.id, n.label, n.qualified_name, n.path, e.id, e.label, e.path, e.line_start, e.line_end " + "LIMIT 100" + ), + parameters=("name",), + returns=( + "node_id", + "node_label", + "node_qualified_name", + "node_path", + "evidence_id", + "evidence_label", + "evidence_path", + "evidence_line_start", + "evidence_line_end", + ), + ), + ), + ), + "graph_quality_gaps": ArchitectureQueryGroup( + name="graph_quality_gaps", + goal="Detect graph gaps that reduce confidence in architecture claims.", + queries=( + ArchitectureQuerySpec( + name="unresolved_reference_risk", + description="Find references without resolved semantic targets.", + statement=( + "MATCH (r:Reference) " + "WHERE NOT EXISTS { MATCH (r)-[:FROM_ResolvesTo]->(:ResolvesTo)-[:TO_ResolvesTo]->() } " + "RETURN r.id, r.label, r.path, r.line_start ORDER BY r.path, r.line_start LIMIT 200" + ), + returns=("id", "label", "path", "line_start"), + ), + ), + ), +} + + +def architecture_query_catalog(group: str | None = None) -> dict[str, Any]: + groups = _selected_groups(group) + return { + "workflow": WORKFLOW_NAME, + "recommended_order": list(ARCHITECTURE_QUERY_ORDER), + "execution_tool": EXECUTION_TOOL, + "groups": [query_group.as_dict() for query_group in groups], + } + + +def _selected_groups(group: str | None) -> tuple[ArchitectureQueryGroup, ...]: + if group is None or group == "": + return tuple(ARCHITECTURE_QUERY_GROUPS[name] for name in ARCHITECTURE_QUERY_ORDER) + try: + return (ARCHITECTURE_QUERY_GROUPS[group],) + except KeyError as exc: + valid = ", ".join(ARCHITECTURE_QUERY_ORDER) + raise ValueError(f"Unknown architecture query group: {group}. Valid groups: {valid}") from exc + + +__all__ = [ + "ARCHITECTURE_QUERY_GROUPS", + "ARCHITECTURE_QUERY_ORDER", + "EXECUTION_TOOL", + "WORKFLOW_NAME", + "ArchitectureQueryGroup", + "ArchitectureQuerySpec", + "architecture_query_catalog", +] diff --git a/src/codebase_graph/setup/instructions.py b/src/codebase_graph/setup/instructions.py index 8e01bf9..0fda6cd 100644 --- a/src/codebase_graph/setup/instructions.py +++ b/src/codebase_graph/setup/instructions.py @@ -73,6 +73,7 @@ def _instruction_block(*, server_name: str, config_path: Path, setup_command: st f"- Treat the `{server_name}` MCP server knowledge graph as the project operating source of truth.\n" f"- Use the `{server_name}` MCP server for repository graph search, schema, and compact context before answering repo-structure questions or performing coding tasks.\n" "- Prefer `graph_search` for symbols, paths, docs, and setup instructions; follow with `graph_context` when relationships or nearby evidence matter.\n" + "- For coding tasks that requires architecture orientation, call `graph_architecture_queries` first, then execute selected statements through `graph_query` that relevant to your task.\n" "- Use `graph_schema` or `graph_query_helpers` before writing raw graph queries, and keep `graph_query` read-only.\n" f"- Refresh the graph with `{setup_command} setup --repo-root . --mcp-client none` when files change materially; install or update MCP with `{setup_command} mcp install --client codex`. Setup config: `{config_path.as_posix()}`.\n" f"{END_MARKER}\n" diff --git a/tests/test_architecture_queries.py b/tests/test_architecture_queries.py new file mode 100644 index 0000000..c72bc03 --- /dev/null +++ b/tests/test_architecture_queries.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +import re + +import pytest + +from codebase_graph.reasoning import ( + ARCHITECTURE_QUERY_GROUPS, + ARCHITECTURE_QUERY_ORDER, + architecture_query_catalog, +) + + +def test_architecture_query_catalog_serializes_in_stable_workflow_order() -> None: + payload = architecture_query_catalog() + + assert payload["workflow"] == "coding_task_architecture_discovery" + assert payload["execution_tool"] == "graph_query" + assert payload["recommended_order"] == list(ARCHITECTURE_QUERY_ORDER) + assert [group["name"] for group in payload["groups"]] == list(ARCHITECTURE_QUERY_ORDER) + + +def test_architecture_query_catalog_groups_expected_queries() -> None: + assert set(ARCHITECTURE_QUERY_GROUPS) == set(ARCHITECTURE_QUERY_ORDER) + assert _query_names("overview") == { + "graph_coverage", + "source_unit_inventory", + "package_directory_shape", + } + assert _query_names("public_surface") == { + "public_surface_candidates", + "entrypoint_runtime_surface", + } + assert _query_names("dependency_topology") == { + "external_dependency_map", + "module_import_coupling", + } + assert _query_names("execution_flow") == { + "high_fan_in_definitions", + "high_fan_out_callers", + "callable_neighborhood", + } + assert _query_names("runtime_data_security") == { + "data_query_touchpoints", + "secret_configuration_touchpoints", + } + assert _query_names("documentation_context") == { + "documentation_to_code_links", + "evidence_for_symbol", + } + assert _query_names("graph_quality_gaps") == {"unresolved_reference_risk"} + + +def test_architecture_query_names_are_unique_and_count_matches_catalog_contract() -> None: + names = [ + query.name + for group_name in ARCHITECTURE_QUERY_ORDER + for query in ARCHITECTURE_QUERY_GROUPS[group_name].queries + ] + + assert len(names) == 15 + assert len(names) == len(set(names)) + + +def test_architecture_query_catalog_filters_by_group() -> None: + payload = architecture_query_catalog("execution_flow") + + assert payload["recommended_order"] == list(ARCHITECTURE_QUERY_ORDER) + assert [group["name"] for group in payload["groups"]] == ["execution_flow"] + + +def test_architecture_query_catalog_rejects_unknown_group() -> None: + with pytest.raises(ValueError, match="Valid groups: overview"): + architecture_query_catalog("missing") + + +def test_architecture_queries_are_read_only_and_use_edge_node_traversal() -> None: + forbidden = re.compile( + r"\b(CREATE|MERGE|DELETE|SET|DROP|LOAD|COPY|INSERT|ALTER|REMOVE|RENAME|DETACH|INSTALL)\b", + re.IGNORECASE, + ) + direct_relation = re.compile(r"-\[:(?!FROM_|TO_)([A-Za-z][A-Za-z0-9_]*)\]->") + + for group_name in ARCHITECTURE_QUERY_ORDER: + for query in ARCHITECTURE_QUERY_GROUPS[group_name].queries: + assert query.statement.lstrip().upper().startswith("MATCH "), query.name + assert ";" not in query.statement, query.name + assert not forbidden.search(query.statement), query.name + assert not direct_relation.search(query.statement), query.name + + +def _query_names(group_name: str) -> set[str]: + return {query.name for query in ARCHITECTURE_QUERY_GROUPS[group_name].queries} diff --git a/tests/test_mcp_portability.py b/tests/test_mcp_portability.py index c6967e6..c690a98 100644 --- a/tests/test_mcp_portability.py +++ b/tests/test_mcp_portability.py @@ -44,6 +44,49 @@ def test_initialize_negotiates_supported_and_fallback_protocol_versions(tmp_path assert "2025-11-25" in SUPPORTED_PROTOCOL_VERSIONS +def test_architecture_query_catalog_is_available_over_mcp_without_opening_graph(tmp_path: Path) -> None: + db_path = tmp_path / "graph.ldb" + db_path.write_text("", encoding="utf-8") + server = McpGraphServer(GraphRuntimeConfig(repo_root=tmp_path, db_path=db_path)) + + listed = server.handle_json_rpc({"jsonrpc": "2.0", "id": 1, "method": "tools/list", "params": {}}) + all_queries = server.handle_json_rpc( + { + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": {"name": "graph_architecture_queries", "arguments": {}}, + } + ) + filtered = server.handle_json_rpc( + { + "jsonrpc": "2.0", + "id": 3, + "method": "tools/call", + "params": {"name": "graph_architecture_queries", "arguments": {"group": "overview"}}, + } + ) + invalid = server.handle_json_rpc( + { + "jsonrpc": "2.0", + "id": 4, + "method": "tools/call", + "params": {"name": "graph_architecture_queries", "arguments": {"group": "missing"}}, + } + ) + + assert listed is not None + assert all_queries is not None + assert filtered is not None + assert invalid is not None + assert any(tool["name"] == "graph_architecture_queries" for tool in listed["result"]["tools"]) + assert all_queries["result"]["structuredContent"]["workflow"] == "coding_task_architecture_discovery" + assert all_queries["result"]["structuredContent"]["execution_tool"] == "graph_query" + assert [group["name"] for group in filtered["result"]["structuredContent"]["groups"]] == ["overview"] + assert invalid["result"]["isError"] is True + assert invalid["result"]["structuredContent"]["error"]["type"] == "ValueError" + + def test_descriptor_prefers_current_environment_script(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: bin_dir = tmp_path / "venv" / "bin" bin_dir.mkdir(parents=True) diff --git a/tests/test_setup_workflow.py b/tests/test_setup_workflow.py index 2558d11..83adc94 100644 --- a/tests/test_setup_workflow.py +++ b/tests/test_setup_workflow.py @@ -54,6 +54,7 @@ def test_setup_cli_creates_state_db_mcp_config_instructions_and_searchable_docs( agents_text = (repo_root / "AGENTS.md").read_text(encoding="utf-8") assert agents_text.count(START_MARKER) == 1 assert agents_text.count(END_MARKER) == 1 + assert "graph_architecture_queries" in agents_text mcp_payload = tomllib.loads(mcp_config_path.read_text(encoding="utf-8")) assert "otherServer" not in mcp_payload.get("mcp_servers", {}) assert mcp_payload["mcp_servers"]["codebase_graph"]["args"] == [ From 2c084a65a31068eedc85e94915ffca6e02101bdd Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Wed, 27 May 2026 10:37:57 +0930 Subject: [PATCH 19/53] feat: implement release process --- .github/workflows/release.yml | 45 ++++++++++++++++++++++++++++------- .release-please-manifest.json | 3 +++ CHANGELOG.md | 3 +++ README.md | 2 +- docs/release.md | 19 +++++++-------- release-please-config.json | 10 ++++++++ 6 files changed, 62 insertions(+), 20 deletions(-) create mode 100644 .release-please-manifest.json create mode 100644 CHANGELOG.md create mode 100644 release-please-config.json diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d2190ef..25e04ad 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,27 +2,46 @@ name: Release on: push: - tags: - - "v*.*.*" + branches: + - main workflow_dispatch: - inputs: - tag: - description: "Release tag to publish, in vX.Y.Z format" - required: true - type: string permissions: contents: read jobs: + release-please: + name: release please + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + outputs: + release-created: ${{ steps.release.outputs.release_created }} + tag-name: ${{ steps.release.outputs.tag_name }} + version: ${{ steps.release.outputs.version }} + + steps: + - name: Create release pull request or GitHub release + id: release + uses: googleapis/release-please-action@v4 + with: + token: ${{ secrets.RELEASE_PLEASE_TOKEN || github.token }} + config-file: release-please-config.json + manifest-file: .release-please-manifest.json + build: name: build release distributions + needs: release-please + if: needs.release-please.outputs.release-created == 'true' runs-on: ubuntu-latest + permissions: + contents: write outputs: package-version: ${{ steps.verify-version.outputs.package-version }} env: - RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref_name }} + RELEASE_TAG: ${{ needs.release-please.outputs.tag-name }} steps: - name: Check out release tag @@ -99,9 +118,17 @@ jobs: path: dist/* if-no-files-found: error + - name: Upload distributions to GitHub release + env: + GH_TOKEN: ${{ github.token }} + run: gh release upload "$RELEASE_TAG" dist/* --clobber + publish-pypi: name: publish to PyPI - needs: build + needs: + - release-please + - build + if: needs.release-please.outputs.release-created == 'true' runs-on: ubuntu-latest environment: name: pypi diff --git a/.release-please-manifest.json b/.release-please-manifest.json new file mode 100644 index 0000000..e18ee07 --- /dev/null +++ b/.release-please-manifest.json @@ -0,0 +1,3 @@ +{ + ".": "0.0.0" +} diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..0017fc7 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,3 @@ +# Changelog + +Release notes are managed by release-please. diff --git a/README.md b/README.md index 1cb0e66..3db160c 100644 --- a/README.md +++ b/README.md @@ -184,7 +184,7 @@ ruff check . ## CI and releases -GitHub Actions runs pytest across Linux, macOS, and Windows for Python 3.10 through 3.14, plus ruff and package-build validation. Releases are driven by `vX.Y.Z` tags, use tag-derived package versions, and publish to PyPI through Trusted Publishing. +GitHub Actions runs pytest across Linux, macOS, and Windows for Python 3.10 through 3.14, plus ruff and package-build validation. Releases are managed by release-please, use tag-derived package versions, create GitHub Releases with distribution assets, and publish to PyPI through Trusted Publishing. Conda distribution uses the conda-forge staged-recipes path rather than direct Anaconda.org uploads. See [docs/release.md](docs/release.md) for the release workflow and conda-forge submission checklist. diff --git a/docs/release.md b/docs/release.md index 043aeba..72c6668 100644 --- a/docs/release.md +++ b/docs/release.md @@ -1,6 +1,6 @@ # Release Process -`codebaseGraph` releases are tag-driven. The GitHub release workflow builds the source distribution and wheel from a `vX.Y.Z` tag, verifies that the package metadata version matches the tag, and publishes to PyPI with Trusted Publishing. +`codebaseGraph` releases are managed by release-please. The release workflow opens and maintains a release pull request from Conventional Commit history. When that release pull request is merged, release-please creates the `vX.Y.Z` tag and GitHub Release, then the same workflow builds the source distribution and wheel from that tag, verifies that the package metadata version matches the tag, attaches the distributions to the GitHub Release, and publishes to PyPI with Trusted Publishing. ## One-time PyPI setup @@ -21,19 +21,18 @@ Pull requests and pushes to `main` or `codex/**` run: - `ruff check .` on Linux. - A package build on Linux with `python -m build`, `twine check`, and console-script smoke tests from the built wheel. -## PyPI release +## Release flow -1. Confirm the release branch is green in CI. -2. Create an intentional release tag: +1. Merge normal pull requests into `main` with Conventional Commit-style titles or squash commit messages such as `feat: add graph query helpers` or `fix: preserve MCP config`. +2. The `Release` workflow opens or updates a release pull request that updates `CHANGELOG.md` and `.release-please-manifest.json`. +3. Review and merge the release pull request when ready to publish. +4. The `Release` workflow creates the `vX.Y.Z` tag and GitHub Release, builds the distributions from that tag, verifies `Version: X.Y.Z`, uploads the distributions to the GitHub Release, and publishes to PyPI from the protected `pypi` environment. - ```bash - git tag vX.Y.Z - git push origin vX.Y.Z - ``` +The package version remains tag-derived through `setuptools_scm`; do not add a static `project.version` field to `pyproject.toml` just for release-please. -3. The `Release` workflow checks out that tag, builds the distributions, verifies `Version: X.Y.Z`, and publishes to PyPI from the protected `pypi` environment. +To force a specific next version, merge a commit whose body contains a `Release-As: X.Y.Z` trailer. -For manual reruns, use the `Release` workflow dispatch input with an existing `vX.Y.Z` tag. +For manual maintenance, rerun or dispatch the `Release` workflow. If CI checks must run on release-please pull requests, configure a `RELEASE_PLEASE_TOKEN` secret backed by a personal access token or GitHub App token; the default `GITHUB_TOKEN` can create the pull request but does not trigger follow-up workflows from its own events. ## Conda-forge release path diff --git a/release-please-config.json b/release-please-config.json new file mode 100644 index 0000000..5495177 --- /dev/null +++ b/release-please-config.json @@ -0,0 +1,10 @@ +{ + "packages": { + ".": { + "release-type": "python", + "package-name": "codebase-graph", + "include-v-in-tag": true, + "changelog-path": "CHANGELOG.md" + } + } +} From 54900a356625edb88dbe8c4ad1206e868f995e0d Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Wed, 27 May 2026 10:38:15 +0930 Subject: [PATCH 20/53] docs: add Git commit convention section --- AGENTS.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index d336baf..cf01ae7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,3 +7,6 @@ - Use `graph_schema` or `graph_query_helpers` before writing raw graph queries, and keep `graph_query` read-only. - Refresh the graph with `/Users/rabii/Projects/Repositories/codebaseGraph/.venv/bin/codebase-graph setup --repo-root . --mcp-client none` when files change materially; install or update MCP with `/Users/rabii/Projects/Repositories/codebaseGraph/.venv/bin/codebase-graph mcp install --client codex`. Setup config: `/Users/rabii/Projects/Repositories/codebaseGraph/.codebaseGraph/config.json`. + +## Git Commit Convention +- Strictly use Conventional Commits 1.0.0 for commit message. From cccad150ba5ff6360da616c74481fd2afbab2e7a Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Wed, 27 May 2026 10:56:29 +0930 Subject: [PATCH 21/53] fix: use correct PyPI URL --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 25e04ad..5fe6f92 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -132,7 +132,7 @@ jobs: runs-on: ubuntu-latest environment: name: pypi - url: https://pypi.org/p/codebase-graph + url: https://pypi.org/p/cbasegraph permissions: id-token: write From f03640124a14f420c09dd4e92236a4ef0541a4ce Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Wed, 27 May 2026 11:55:21 +0930 Subject: [PATCH 22/53] refactor: update CLI commands and documentation for graph workflow --- AGENTS.md | 12 +- README.md | 16 ++- src/codebase_graph/cli/__init__.py | 133 +++++++++++++++++++++- src/codebase_graph/setup/instructions.py | 12 +- tests/test_search.py | 136 +++++++++++++++++++++++ tests/test_setup_workflow.py | 7 +- 6 files changed, 298 insertions(+), 18 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index cf01ae7..d17471c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,11 +1,11 @@ ## codebaseGraph workflow -- Treat the `codebase_graph` MCP server knowledge graph as the project operating source of truth. -- Use the `codebase_graph` MCP server for repository graph search, schema, and compact context before answering repo-structure questions or performing coding tasks. -- Prefer `graph_search` for symbols, paths, docs, and setup instructions; follow with `graph_context` when relationships or nearby evidence matter. -- For coding tasks that requires architecture orientation, call `graph_architecture_queries` first, then execute selected statements through `graph_query` that relevant to your task. -- Use `graph_schema` or `graph_query_helpers` before writing raw graph queries, and keep `graph_query` read-only. -- Refresh the graph with `/Users/rabii/Projects/Repositories/codebaseGraph/.venv/bin/codebase-graph setup --repo-root . --mcp-client none` when files change materially; install or update MCP with `/Users/rabii/Projects/Repositories/codebaseGraph/.venv/bin/codebase-graph mcp install --client codex`. Setup config: `/Users/rabii/Projects/Repositories/codebaseGraph/.codebaseGraph/config.json`. +- Treat the repo-local `.codebaseGraph` graph as the project operating source of truth. +- Use `/Users/rabii/Projects/Repositories/codebaseGraph/.venv/bin/codebase-graph graph-search --repo-root . --no-refresh --json` before answering repo-structure questions or performing coding tasks. +- Use `/Users/rabii/Projects/Repositories/codebaseGraph/.venv/bin/codebase-graph graph-context --repo-root . --profile --no-refresh --json` when relationships or nearby evidence matter; useful profiles include `definitions`, `dependencies`, `callgraph`, `docs`, `runtime`, and `change_impact`. +- For architecture orientation, run `/Users/rabii/Projects/Repositories/codebaseGraph/.venv/bin/codebase-graph graph-architecture-queries`, then execute selected read-only statements with `/Users/rabii/Projects/Repositories/codebaseGraph/.venv/bin/codebase-graph graph-query "" --repo-root .`. +- Use `/Users/rabii/Projects/Repositories/codebaseGraph/.venv/bin/codebase-graph graph-schema` or `/Users/rabii/Projects/Repositories/codebaseGraph/.venv/bin/codebase-graph graph-query-helpers` before writing raw graph queries, and keep `graph-query` read-only. +- Refresh the graph with `/Users/rabii/Projects/Repositories/codebaseGraph/.venv/bin/codebase-graph setup --repo-root . --mcp-client none` when files change materially. Setup config: `/Users/rabii/Projects/Repositories/codebaseGraph/.codebaseGraph/config.json`. ## Git Commit Convention diff --git a/README.md b/README.md index 3db160c..5f0d526 100644 --- a/README.md +++ b/README.md @@ -158,9 +158,21 @@ Available MCP tools: For coding-task architecture orientation, call `graph_architecture_queries` first to fetch the grouped read-only Cypher catalog, then run selected statements with `graph_query`. -## CLI search +## CLI graph workflow -The legacy materializer/search commands are still available. Setup reports the explicit database and manifest paths to use with them: +The CLI exposes the same graph workflow as the MCP tools, which is useful in clients that do not surface MCP tools directly: + +```bash +codebase-graph graph-health --repo-root . +codebase-graph graph-search SampleService --repo-root . --no-refresh --json +codebase-graph graph-context SampleService --repo-root . --profile definitions --no-refresh --json +codebase-graph graph-schema +codebase-graph graph-query-helpers +codebase-graph graph-architecture-queries --group overview +codebase-graph graph-query "MATCH (n) RETURN count(n) AS total_nodes LIMIT 1" --repo-root . +``` + +`graph-query` blocks write-like statements and should be used read-only. The older `search` and `context` commands remain available. Setup reports the explicit database and manifest paths to use with them when needed: ```bash codebase-graph search SampleService \ diff --git a/src/codebase_graph/cli/__init__.py b/src/codebase_graph/cli/__init__.py index c272372..4d5c6e6 100644 --- a/src/codebase_graph/cli/__init__.py +++ b/src/codebase_graph/cli/__init__.py @@ -7,7 +7,10 @@ from codebase_graph.db import create_ladybug_database from codebase_graph.ingest import GraphMaterializer -from codebase_graph.ontology import CONTEXT_PROFILES +from codebase_graph.mcp.runtime import runtime_config +from codebase_graph.mcp.tools import handle_tool_call +from codebase_graph.ontology import CONTEXT_PROFILES, QUERY_HELPERS, schema_payload +from codebase_graph.reasoning import architecture_query_catalog from codebase_graph.retrieval import SearchRequest, SearchService from codebase_graph.setup import SetupError, SetupOptions, run_setup from codebase_graph.setup.clients import supported_client_ids @@ -31,6 +34,38 @@ def main(argv: Sequence[str] | None = None) -> int: context_parser = subparsers.add_parser("context", help="Return compact context for a search query") _add_search_arguments(context_parser) + graph_health_parser = subparsers.add_parser("graph-health", help="Check configured graph paths") + _add_runtime_arguments(graph_health_parser) + + graph_search_parser = subparsers.add_parser("graph-search", help="Search the code graph with compact context") + graph_search_parser.add_argument("query", help="Search query") + _add_compact_context_arguments(graph_search_parser) + _add_runtime_arguments(graph_search_parser) + _add_graph_compatibility_arguments(graph_search_parser) + + graph_context_parser = subparsers.add_parser("graph-context", help="Return compact graph context") + graph_context_parser.add_argument("query", nargs="?", help="Search query") + graph_context_parser.add_argument("--node-id", default=None, help="Explicit graph node id") + graph_context_parser.add_argument("--node-type", default=None, help="Explicit graph node type") + _add_compact_context_arguments(graph_context_parser) + _add_runtime_arguments(graph_context_parser) + _add_graph_compatibility_arguments(graph_context_parser) + + subparsers.add_parser("graph-schema", help="Return ontology schema, indexes, profiles, and helpers") + subparsers.add_parser("graph-query-helpers", help="Return named read-only graph query helpers") + + graph_architecture_parser = subparsers.add_parser( + "graph-architecture-queries", + help="Return the architecture-discovery query catalog", + ) + graph_architecture_parser.add_argument("--group", default=None, help="Optional architecture query group") + + graph_query_parser = subparsers.add_parser("graph-query", help="Execute a restricted read-only graph query") + graph_query_parser.add_argument("statement", help="Read-only graph query statement") + graph_query_parser.add_argument("--parameters", default="{}", help="JSON object with query parameters") + graph_query_parser.add_argument("--limit", type=int, default=100, help="Maximum rows to return") + _add_runtime_arguments(graph_query_parser) + setup_parser = subparsers.add_parser("setup", help="Bootstrap codebaseGraph state for a repository") setup_parser.add_argument("--repo-root", default=".", help="Repository root to configure") setup_parser.add_argument("--mcp-client", choices=supported_client_ids(), default="codex") @@ -115,6 +150,46 @@ def main(argv: Sequence[str] | None = None) -> int: materializer.close() print(json.dumps(payload.as_dict(), indent=2, sort_keys=True)) return 0 + if args.command == "graph-health": + return _print_tool_payload(parser, "graph_health", {}, args) + if args.command == "graph-search": + return _print_tool_payload(parser, "graph_search", _search_arguments_payload(args), args) + if args.command == "graph-context": + if not args.query and not (args.node_id and args.node_type): + parser.error("graph-context requires a query or both --node-id and --node-type") + if (args.node_id and not args.node_type) or (args.node_type and not args.node_id): + parser.error("graph-context explicit lookup requires both --node-id and --node-type") + payload = _search_arguments_payload(args) + if args.node_id and args.node_type: + payload["node_id"] = args.node_id + payload["node_type"] = args.node_type + return _print_tool_payload(parser, "graph_context", payload, args) + if args.command == "graph-schema": + print(json.dumps(schema_payload(), indent=2, sort_keys=True)) + return 0 + if args.command == "graph-query-helpers": + print(json.dumps({"query_helpers": [helper.as_dict() for helper in QUERY_HELPERS]}, indent=2, sort_keys=True)) + return 0 + if args.command == "graph-architecture-queries": + try: + payload = architecture_query_catalog(group=args.group) + except ValueError as exc: + parser.error(str(exc)) + print(json.dumps(payload, indent=2, sort_keys=True)) + return 0 + if args.command == "graph-query": + try: + parameters = json.loads(args.parameters) + except json.JSONDecodeError as exc: + parser.error(f"graph-query --parameters must be a JSON object: {exc}") + if not isinstance(parameters, dict): + parser.error("graph-query --parameters must be a JSON object") + return _print_tool_payload( + parser, + "graph_query", + {"statement": args.statement, "parameters": parameters, "limit": args.limit}, + args, + ) if args.command == "setup": try: result = run_setup( @@ -188,12 +263,64 @@ def _add_search_arguments(parser: argparse.ArgumentParser) -> None: parser.add_argument("--source-root", default=".", help="Repository or source root to search") parser.add_argument("--db", default=None, help="LadybugDB path; defaults under .codebaseGraph") parser.add_argument("--manifest", default=None, help="Manifest path; defaults under .codebaseGraph") + _add_compact_context_arguments(parser) + parser.add_argument("--no-refresh", action="store_true", help="Query the existing graph without changed materialization") + parser.add_argument("--json", action="store_true", help="Emit compact JSON output") + + +def _add_compact_context_arguments(parser: argparse.ArgumentParser) -> None: parser.add_argument("--limit", type=int, default=3, help="Maximum search hits to return") parser.add_argument("--profile", choices=sorted(CONTEXT_PROFILES), default="brief", help="Context profile") parser.add_argument("--budget", type=int, default=600, help="Approximate per-hit context character budget") parser.add_argument("--max-depth", type=int, default=None, help="Override the context profile depth") - parser.add_argument("--no-refresh", action="store_true", help="Query the existing graph without changed materialization") - parser.add_argument("--json", action="store_true", help="Emit compact JSON output") + + +def _add_runtime_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--repo-root", default=".", help="Repository root containing .codebaseGraph/config.json") + parser.add_argument("--config", default=None, help="Path to .codebaseGraph/config.json") + parser.add_argument("--db", default=None, help="Override LadyBugDB path") + parser.add_argument("--manifest", default=None, help="Override manifest path") + + +def _add_graph_compatibility_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--no-refresh", action="store_true", help="Accepted for search/context command parity") + parser.add_argument("--json", action="store_true", help="Accepted for search/context command parity") + + +def _runtime(args: argparse.Namespace) -> object: + return runtime_config( + repo_root=args.repo_root, + config_path=args.config, + db_path=args.db, + manifest_path=args.manifest, + ) + + +def _search_arguments_payload(args: argparse.Namespace) -> dict[str, object]: + payload: dict[str, object] = { + "limit": args.limit, + "profile": args.profile, + "budget": args.budget, + } + if args.query: + payload["query"] = args.query + if args.max_depth is not None: + payload["max_depth"] = args.max_depth + return payload + + +def _print_tool_payload( + parser: argparse.ArgumentParser, + tool_name: str, + arguments: dict[str, object], + args: argparse.Namespace, +) -> int: + try: + payload = handle_tool_call(tool_name, arguments, runtime=_runtime(args)) + except (OSError, ValueError) as exc: + parser.error(str(exc)) + print(json.dumps(payload, indent=2, sort_keys=True)) + return 0 def _result_payload(result: object) -> dict[str, object]: diff --git a/src/codebase_graph/setup/instructions.py b/src/codebase_graph/setup/instructions.py index 0fda6cd..0cc7388 100644 --- a/src/codebase_graph/setup/instructions.py +++ b/src/codebase_graph/setup/instructions.py @@ -70,12 +70,12 @@ def _instruction_block(*, server_name: str, config_path: Path, setup_command: st return ( f"{START_MARKER}\n" "## codebaseGraph workflow\n" - f"- Treat the `{server_name}` MCP server knowledge graph as the project operating source of truth.\n" - f"- Use the `{server_name}` MCP server for repository graph search, schema, and compact context before answering repo-structure questions or performing coding tasks.\n" - "- Prefer `graph_search` for symbols, paths, docs, and setup instructions; follow with `graph_context` when relationships or nearby evidence matter.\n" - "- For coding tasks that requires architecture orientation, call `graph_architecture_queries` first, then execute selected statements through `graph_query` that relevant to your task.\n" - "- Use `graph_schema` or `graph_query_helpers` before writing raw graph queries, and keep `graph_query` read-only.\n" - f"- Refresh the graph with `{setup_command} setup --repo-root . --mcp-client none` when files change materially; install or update MCP with `{setup_command} mcp install --client codex`. Setup config: `{config_path.as_posix()}`.\n" + "- Treat the repo-local `.codebaseGraph` graph as the project operating source of truth.\n" + f"- Use `{setup_command} graph-search --repo-root . --no-refresh --json` before answering repo-structure questions or performing coding tasks.\n" + f"- Use `{setup_command} graph-context --repo-root . --profile --no-refresh --json` when relationships or nearby evidence matter; useful profiles include `definitions`, `dependencies`, `callgraph`, `docs`, `runtime`, and `change_impact`.\n" + f"- For architecture orientation, run `{setup_command} graph-architecture-queries`, then execute selected read-only statements with `{setup_command} graph-query \"\" --repo-root .`.\n" + f"- Use `{setup_command} graph-schema` or `{setup_command} graph-query-helpers` before writing raw graph queries, and keep `graph-query` read-only.\n" + f"- Refresh the graph with `{setup_command} setup --repo-root . --mcp-client none` when files change materially. Setup config: `{config_path.as_posix()}`.\n" f"{END_MARKER}\n" ) diff --git a/tests/test_search.py b/tests/test_search.py index 528cc69..685998d 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -10,6 +10,8 @@ from codebase_graph.cli import main as cli_main from codebase_graph.db import GraphNeighbor, SearchIndexRow from codebase_graph.ingest import GraphMaterializer +from codebase_graph.mcp.runtime import GraphRuntimeConfig +from codebase_graph.mcp.tools import handle_tool_call from codebase_graph.reasoning import CompactContextBuilder from codebase_graph.retrieval.search import SearchHit, SearchRequest, SearchService @@ -361,6 +363,140 @@ def test_cli_search_and_context_return_compact_json_without_refresh(tmp_path: Pa assert any(hit["label"] == "helper" and hit["context"] for hit in context_payload["results"]) +def test_cli_graph_commands_match_mcp_tool_payloads(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + _require_graph_runtime() + source_root = _copy_fixture(tmp_path) + db_path = tmp_path / "graph.lbug" + manifest_path = tmp_path / "manifest.json" + + assert cli_main([ + "materialize", + "--source-root", + source_root.as_posix(), + "--db", + db_path.as_posix(), + "--manifest", + manifest_path.as_posix(), + "--mode", + "full", + ]) == 0 + capsys.readouterr() + runtime = GraphRuntimeConfig(repo_root=source_root, db_path=db_path, manifest_path=manifest_path) + + assert cli_main([ + "graph-health", + "--repo-root", + source_root.as_posix(), + "--db", + db_path.as_posix(), + "--manifest", + manifest_path.as_posix(), + ]) == 0 + assert json.loads(capsys.readouterr().out) == handle_tool_call("graph_health", {}, runtime=runtime) + + search_args = {"query": "SampleService", "limit": 2, "profile": "brief", "budget": 600} + assert cli_main([ + "graph-search", + "SampleService", + "--repo-root", + source_root.as_posix(), + "--db", + db_path.as_posix(), + "--manifest", + manifest_path.as_posix(), + "--limit", + "2", + "--no-refresh", + "--json", + ]) == 0 + search_payload = json.loads(capsys.readouterr().out) + assert search_payload == handle_tool_call("graph_search", search_args, runtime=runtime) + + hit = next(item for item in search_payload["results"] if item["label"] == "SampleService") + context_args = { + "node_id": hit["id"], + "node_type": hit["type"], + "limit": 1, + "profile": "definitions", + "budget": 600, + } + assert cli_main([ + "graph-context", + "--node-id", + hit["id"], + "--node-type", + hit["type"], + "--repo-root", + source_root.as_posix(), + "--db", + db_path.as_posix(), + "--manifest", + manifest_path.as_posix(), + "--profile", + "definitions", + "--limit", + "1", + ]) == 0 + assert json.loads(capsys.readouterr().out) == handle_tool_call("graph_context", context_args, runtime=runtime) + + statement = "MATCH (n) RETURN count(n) AS total_nodes LIMIT 1" + query_args = {"statement": statement, "parameters": {}, "limit": 5} + assert cli_main([ + "graph-query", + statement, + "--repo-root", + source_root.as_posix(), + "--db", + db_path.as_posix(), + "--manifest", + manifest_path.as_posix(), + "--limit", + "5", + ]) == 0 + assert json.loads(capsys.readouterr().out) == handle_tool_call("graph_query", query_args, runtime=runtime) + + +def test_cli_graph_metadata_commands_do_not_open_graph_db(capsys: pytest.CaptureFixture[str]) -> None: + assert cli_main(["graph-schema"]) == 0 + schema = json.loads(capsys.readouterr().out) + assert schema["ontology"] + assert schema["context_profiles"] + + assert cli_main(["graph-query-helpers"]) == 0 + helpers = json.loads(capsys.readouterr().out) + assert any(helper["name"] == "repository_overview" for helper in helpers["query_helpers"]) + + assert cli_main(["graph-architecture-queries", "--group", "overview"]) == 0 + architecture = json.loads(capsys.readouterr().out) + assert [group["name"] for group in architecture["groups"]] == ["overview"] + + +def test_cli_graph_query_rejects_write_like_statements(tmp_path: Path) -> None: + _require_graph_runtime() + source_root = _copy_fixture(tmp_path) + db_path = tmp_path / "graph.lbug" + manifest_path = tmp_path / "manifest.json" + materializer = GraphMaterializer(source_root, db_path=db_path, manifest_path=manifest_path) + try: + materializer.materialize(mode="full") + finally: + materializer.close() + + with pytest.raises(SystemExit) as exc_info: + cli_main([ + "graph-query", + "MATCH (n) DELETE n", + "--repo-root", + source_root.as_posix(), + "--db", + db_path.as_posix(), + "--manifest", + manifest_path.as_posix(), + ]) + + assert exc_info.value.code == 2 + + def _require_graph_runtime() -> None: pytest.importorskip("tree_sitter") pytest.importorskip("tree_sitter_python") diff --git a/tests/test_setup_workflow.py b/tests/test_setup_workflow.py index 83adc94..bfc6f9e 100644 --- a/tests/test_setup_workflow.py +++ b/tests/test_setup_workflow.py @@ -54,7 +54,12 @@ def test_setup_cli_creates_state_db_mcp_config_instructions_and_searchable_docs( agents_text = (repo_root / "AGENTS.md").read_text(encoding="utf-8") assert agents_text.count(START_MARKER) == 1 assert agents_text.count(END_MARKER) == 1 - assert "graph_architecture_queries" in agents_text + assert "graph-search" in agents_text + assert "graph-context" in agents_text + assert "graph-architecture-queries" in agents_text + assert "MCP server" not in agents_text + assert "graph_architecture_queries" not in agents_text + assert "graph_query" not in agents_text mcp_payload = tomllib.loads(mcp_config_path.read_text(encoding="utf-8")) assert "otherServer" not in mcp_payload.get("mcp_servers", {}) assert mcp_payload["mcp_servers"]["codebase_graph"]["args"] == [ From b1dd5b80f6ad8f82584a82b3a76178ae3d15b731 Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Wed, 27 May 2026 12:40:27 +0930 Subject: [PATCH 23/53] feat: reduce graph retrieval output token usage Add shared detail and context fanout controls across CLI and MCP retrieval paths while switching CLI JSON output to compact defaults. The standard detail mode preserves existing payload shape, and slim mode is opt-in for lower-token agent workflows. Constraint: Existing standard retrieval payloads must remain backward compatible Constraint: CLI and MCP graph retrieval outputs must stay aligned Rejected: Make slim the default | too risky for existing consumers that inspect score diagnostics Confidence: high Scope-risk: moderate Directive: Do not remove standard detail fields without adding a compatibility path Tested: .venv/bin/ruff check . Tested: .venv/bin/pytest tests/test_search.py tests/test_mcp_portability.py tests/test_setup_workflow.py Tested: .venv/bin/pytest Not-tested: Downstream third-party MCP clients beyond local protocol tests --- README.md | 6 +- src/codebase_graph/cli/__init__.py | 49 ++++++-- src/codebase_graph/mcp/tools.py | 24 +++- .../reasoning/context_builder.py | 18 ++- src/codebase_graph/retrieval/__init__.py | 4 +- src/codebase_graph/retrieval/search.py | 48 +++++++- src/codebase_graph/setup/instructions.py | 6 +- tests/test_mcp_portability.py | 4 + tests/test_search.py | 105 +++++++++++++++++- 9 files changed, 232 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 5f0d526..5aa8d3b 100644 --- a/README.md +++ b/README.md @@ -164,14 +164,16 @@ The CLI exposes the same graph workflow as the MCP tools, which is useful in cli ```bash codebase-graph graph-health --repo-root . -codebase-graph graph-search SampleService --repo-root . --no-refresh --json -codebase-graph graph-context SampleService --repo-root . --profile definitions --no-refresh --json +codebase-graph graph-search SampleService --repo-root . --no-refresh --detail slim --context-limit 1 --json +codebase-graph graph-context SampleService --repo-root . --profile definitions --no-refresh --detail slim --context-limit 2 --json codebase-graph graph-schema codebase-graph graph-query-helpers codebase-graph graph-architecture-queries --group overview codebase-graph graph-query "MATCH (n) RETURN count(n) AS total_nodes LIMIT 1" --repo-root . ``` +CLI JSON output is minified by default to reduce tokens. Add `--pretty` to JSON-producing commands when you want indented output. Retrieval commands support `--detail standard|slim`; `standard` keeps the full payload, while `slim` drops score diagnostics and duplicate or empty summary fields. + `graph-query` blocks write-like statements and should be used read-only. The older `search` and `context` commands remain available. Setup reports the explicit database and manifest paths to use with them when needed: ```bash diff --git a/src/codebase_graph/cli/__init__.py b/src/codebase_graph/cli/__init__.py index 4d5c6e6..2669243 100644 --- a/src/codebase_graph/cli/__init__.py +++ b/src/codebase_graph/cli/__init__.py @@ -27,6 +27,7 @@ def main(argv: Sequence[str] | None = None) -> int: materialize_parser.add_argument("--manifest", default=None, help="Manifest path; defaults under .codebaseGraph") materialize_parser.add_argument("--mode", choices=("full", "changed"), default="changed") materialize_parser.add_argument("--no-fts", action="store_true", help="Skip FTS index creation") + _add_json_output_arguments(materialize_parser) search_parser = subparsers.add_parser("search", help="Search the code graph with compact context") _add_search_arguments(search_parser) @@ -36,6 +37,7 @@ def main(argv: Sequence[str] | None = None) -> int: graph_health_parser = subparsers.add_parser("graph-health", help="Check configured graph paths") _add_runtime_arguments(graph_health_parser) + _add_json_output_arguments(graph_health_parser) graph_search_parser = subparsers.add_parser("graph-search", help="Search the code graph with compact context") graph_search_parser.add_argument("query", help="Search query") @@ -51,20 +53,24 @@ def main(argv: Sequence[str] | None = None) -> int: _add_runtime_arguments(graph_context_parser) _add_graph_compatibility_arguments(graph_context_parser) - subparsers.add_parser("graph-schema", help="Return ontology schema, indexes, profiles, and helpers") - subparsers.add_parser("graph-query-helpers", help="Return named read-only graph query helpers") + graph_schema_parser = subparsers.add_parser("graph-schema", help="Return ontology schema, indexes, profiles, and helpers") + _add_json_output_arguments(graph_schema_parser) + graph_query_helpers_parser = subparsers.add_parser("graph-query-helpers", help="Return named read-only graph query helpers") + _add_json_output_arguments(graph_query_helpers_parser) graph_architecture_parser = subparsers.add_parser( "graph-architecture-queries", help="Return the architecture-discovery query catalog", ) graph_architecture_parser.add_argument("--group", default=None, help="Optional architecture query group") + _add_json_output_arguments(graph_architecture_parser) graph_query_parser = subparsers.add_parser("graph-query", help="Execute a restricted read-only graph query") graph_query_parser.add_argument("statement", help="Read-only graph query statement") graph_query_parser.add_argument("--parameters", default="{}", help="JSON object with query parameters") graph_query_parser.add_argument("--limit", type=int, default=100, help="Maximum rows to return") _add_runtime_arguments(graph_query_parser) + _add_json_output_arguments(graph_query_parser) setup_parser = subparsers.add_parser("setup", help="Bootstrap codebaseGraph state for a repository") setup_parser.add_argument("--repo-root", default=".", help="Repository root to configure") @@ -80,6 +86,7 @@ def main(argv: Sequence[str] | None = None) -> int: ) setup_parser.add_argument("--mode", choices=("full", "changed"), default="changed", help="Materialization mode") setup_parser.add_argument("--json", action="store_true", help="Emit JSON output") + _add_json_output_arguments(setup_parser) mcp_parser = subparsers.add_parser("mcp", help="Run or inspect the MCP server") mcp_subparsers = mcp_parser.add_subparsers(dest="mcp_command", required=True) @@ -92,6 +99,7 @@ def main(argv: Sequence[str] | None = None) -> int: install_parser.add_argument("--dry-run", action="store_true", help="Show the install action without writing or invoking CLIs") install_parser.add_argument("--verify", action="store_true", help="Run direct MCP smoke checks after installation") install_parser.add_argument("--json", action="store_true", help="Emit JSON output") + _add_json_output_arguments(install_parser) serve_parser = mcp_subparsers.add_parser("serve", help="Serve graph tools over MCP stdio") serve_parser.add_argument("--repo-root", default=".", help="Repository root containing .codebaseGraph/config.json") @@ -119,7 +127,7 @@ def main(argv: Sequence[str] | None = None) -> int: result = materializer.materialize(mode=args.mode) finally: materializer.close() - print(json.dumps(_result_payload(result), indent=2, sort_keys=True)) + _print_json(_result_payload(result), args) return 0 if args.command in {"search", "context"}: request = SearchRequest( @@ -128,6 +136,8 @@ def main(argv: Sequence[str] | None = None) -> int: profile=args.profile, budget=args.budget, max_depth=args.max_depth, + context_limit=args.context_limit, + detail=args.detail, ) try: request.validate() @@ -148,7 +158,7 @@ def main(argv: Sequence[str] | None = None) -> int: payload = SearchService(materializer.store).search(request) finally: materializer.close() - print(json.dumps(payload.as_dict(), indent=2, sort_keys=True)) + _print_json(payload.as_dict(detail=args.detail), args) return 0 if args.command == "graph-health": return _print_tool_payload(parser, "graph_health", {}, args) @@ -165,17 +175,17 @@ def main(argv: Sequence[str] | None = None) -> int: payload["node_type"] = args.node_type return _print_tool_payload(parser, "graph_context", payload, args) if args.command == "graph-schema": - print(json.dumps(schema_payload(), indent=2, sort_keys=True)) + _print_json(schema_payload(), args) return 0 if args.command == "graph-query-helpers": - print(json.dumps({"query_helpers": [helper.as_dict() for helper in QUERY_HELPERS]}, indent=2, sort_keys=True)) + _print_json({"query_helpers": [helper.as_dict() for helper in QUERY_HELPERS]}, args) return 0 if args.command == "graph-architecture-queries": try: payload = architecture_query_catalog(group=args.group) except ValueError as exc: parser.error(str(exc)) - print(json.dumps(payload, indent=2, sort_keys=True)) + _print_json(payload, args) return 0 if args.command == "graph-query": try: @@ -205,7 +215,7 @@ def main(argv: Sequence[str] | None = None) -> int: ) except SetupError as exc: parser.error(str(exc)) - print(json.dumps(result.as_dict(), indent=2, sort_keys=True)) + _print_json(result.as_dict(), args) return 0 if args.command == "mcp" and args.mcp_command == "install": setup_config_path = ( @@ -232,7 +242,7 @@ def main(argv: Sequence[str] | None = None) -> int: else: payload = results[0].as_dict() if args.json: - print(json.dumps(payload, indent=2, sort_keys=True)) + _print_json(payload, args) else: _print_mcp_install_results(results) return 1 if any(result.action == "failed" for result in results) else 0 @@ -273,6 +283,9 @@ def _add_compact_context_arguments(parser: argparse.ArgumentParser) -> None: parser.add_argument("--profile", choices=sorted(CONTEXT_PROFILES), default="brief", help="Context profile") parser.add_argument("--budget", type=int, default=600, help="Approximate per-hit context character budget") parser.add_argument("--max-depth", type=int, default=None, help="Override the context profile depth") + parser.add_argument("--context-limit", type=int, default=3, help="Maximum context items per search hit") + parser.add_argument("--detail", choices=("standard", "slim"), default="standard", help="Output detail level") + _add_json_output_arguments(parser) def _add_runtime_arguments(parser: argparse.ArgumentParser) -> None: @@ -301,6 +314,8 @@ def _search_arguments_payload(args: argparse.Namespace) -> dict[str, object]: "limit": args.limit, "profile": args.profile, "budget": args.budget, + "context_limit": args.context_limit, + "detail": args.detail, } if args.query: payload["query"] = args.query @@ -319,10 +334,24 @@ def _print_tool_payload( payload = handle_tool_call(tool_name, arguments, runtime=_runtime(args)) except (OSError, ValueError) as exc: parser.error(str(exc)) - print(json.dumps(payload, indent=2, sort_keys=True)) + _print_json(payload, args) return 0 +def _add_json_output_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--pretty", action="store_true", help="Emit indented JSON output") + + +def _print_json(payload: object, args: argparse.Namespace) -> None: + print(_json_dumps(payload, pretty=getattr(args, "pretty", False))) + + +def _json_dumps(payload: object, *, pretty: bool) -> str: + if pretty: + return json.dumps(payload, indent=2, sort_keys=True) + return json.dumps(payload, separators=(",", ":"), sort_keys=True) + + def _result_payload(result: object) -> dict[str, object]: return { "mode": getattr(result, "mode"), diff --git a/src/codebase_graph/mcp/tools.py b/src/codebase_graph/mcp/tools.py index 89f1239..43c7950 100644 --- a/src/codebase_graph/mcp/tools.py +++ b/src/codebase_graph/mcp/tools.py @@ -7,7 +7,7 @@ from codebase_graph.db import LadybugCodeGraphStore from codebase_graph.ontology import QUERY_HELPERS, schema_payload from codebase_graph.reasoning import CompactContextBuilder, architecture_query_catalog -from codebase_graph.retrieval import SearchRequest, SearchService +from codebase_graph.retrieval import DETAIL_LEVELS, SearchRequest, SearchService from .runtime import GraphRuntimeConfig, open_graph_store @@ -33,7 +33,7 @@ def handle_tool_call(name: str, arguments: dict[str, Any], *, runtime: GraphRunt if name == "graph_search": with open_graph_store(runtime) as store: request = _search_request(arguments) - return SearchService(store).search(request).as_dict() + return SearchService(store).search(request).as_dict(detail=request.detail) if name == "graph_context": with open_graph_store(runtime) as store: return _context_payload(store, arguments) @@ -55,7 +55,7 @@ def call_tool_result(name: str, arguments: dict[str, Any], *, runtime: GraphRunt def tool_result(payload: dict[str, Any]) -> dict[str, Any]: return { - "content": [{"type": "text", "text": json.dumps(payload, indent=2, sort_keys=True)}], + "content": [{"type": "text", "text": json.dumps(payload, separators=(",", ":"), sort_keys=True)}], "structuredContent": payload, "isError": False, } @@ -150,6 +150,8 @@ def _search_request(arguments: dict[str, Any]) -> SearchRequest: profile=str(arguments.get("profile", "brief")), budget=int(arguments.get("budget", 600)), max_depth=_optional_int(arguments.get("max_depth")), + context_limit=int(arguments.get("context_limit", 3)), + detail=_detail(arguments), ) request.validate() return request @@ -160,6 +162,7 @@ def _context_payload(store: LadybugCodeGraphStore, arguments: dict[str, Any]) -> node_type = str(arguments.get("node_type") or "") if node_id and node_type: profile = str(arguments.get("profile", "brief")) + detail = _detail(arguments) context = CompactContextBuilder(store).build( node_id, node_type, @@ -172,9 +175,10 @@ def _context_payload(store: LadybugCodeGraphStore, arguments: dict[str, Any]) -> "node_id": node_id, "node_type": node_type, "profile": profile, - "context": [node.as_dict() for node in context], + "context": [node.as_dict(detail=detail) for node in context], } - return SearchService(store).search(_search_request(arguments)).as_dict() + request = _search_request(arguments) + return SearchService(store).search(request).as_dict(detail=request.detail) def _query_payload(store: LadybugCodeGraphStore, arguments: dict[str, Any]) -> dict[str, Any]: @@ -230,6 +234,8 @@ def _search_schema(*, required: tuple[str, ...]) -> dict[str, Any]: "profile": {"type": "string"}, "budget": {"type": "integer", "minimum": 0}, "max_depth": {"type": "integer", "minimum": 0}, + "context_limit": {"type": "integer", "minimum": 0}, + "detail": {"type": "string", "enum": sorted(DETAIL_LEVELS)}, "node_id": {"type": "string"}, "node_type": {"type": "string"}, }, @@ -248,3 +254,11 @@ def _optional_str(value: Any) -> str | None: if value is None or value == "": return None return str(value) + + +def _detail(arguments: dict[str, Any]) -> str: + detail = str(arguments.get("detail", "standard")) + if detail not in DETAIL_LEVELS: + valid = ", ".join(sorted(DETAIL_LEVELS)) + raise ValueError(f"Unknown detail level: {detail}. Valid levels: {valid}") + return detail diff --git a/src/codebase_graph/reasoning/context_builder.py b/src/codebase_graph/reasoning/context_builder.py index ddac2df..82d1484 100644 --- a/src/codebase_graph/reasoning/context_builder.py +++ b/src/codebase_graph/reasoning/context_builder.py @@ -22,7 +22,23 @@ class ContextNode: summary: str = "" id: str = field(default="", repr=False) - def as_dict(self) -> dict[str, Any]: + def as_dict(self, *, detail: str = "standard") -> dict[str, Any]: + if detail not in {"standard", "slim"}: + raise ValueError(f"Unknown detail level: {detail}. Valid levels: slim, standard") + if detail == "slim": + payload: dict[str, Any] = { + "relation": self.relation, + "direction": self.direction, + "type": self.type, + "label": self.label, + } + if self.path: + payload["path"] = self.path + if self.span: + payload["span"] = dict(self.span) + if self.summary and self.summary != self.label: + payload["summary"] = self.summary + return payload return { "relation": self.relation, "direction": self.direction, diff --git a/src/codebase_graph/retrieval/__init__.py b/src/codebase_graph/retrieval/__init__.py index cd58222..e204c8a 100644 --- a/src/codebase_graph/retrieval/__init__.py +++ b/src/codebase_graph/retrieval/__init__.py @@ -1,5 +1,5 @@ """Keyword, vector, graph traversal, and ranking retrieval.""" -from .search import CompactContextPayload, SearchHit, SearchRequest, SearchService +from .search import DETAIL_LEVELS, CompactContextPayload, SearchHit, SearchRequest, SearchService -__all__ = ["CompactContextPayload", "SearchHit", "SearchRequest", "SearchService"] +__all__ = ["DETAIL_LEVELS", "CompactContextPayload", "SearchHit", "SearchRequest", "SearchService"] diff --git a/src/codebase_graph/retrieval/search.py b/src/codebase_graph/retrieval/search.py index e6724e2..73645f7 100644 --- a/src/codebase_graph/retrieval/search.py +++ b/src/codebase_graph/retrieval/search.py @@ -11,6 +11,7 @@ DEFAULT_SEARCH_LIMIT = 3 MAX_CANDIDATE_LIMIT = 50 MIN_CANDIDATE_LIMIT = 10 +DETAIL_LEVELS = {"standard", "slim"} DEFINITION_TYPES = {"Class", "Function", "Method", "Variable", "Constant"} GENERIC_TYPES = {"Symbol", "Dependency"} @@ -22,6 +23,8 @@ class SearchRequest: profile: str = "brief" budget: int = DEFAULT_CONTEXT_BUDGET max_depth: int | None = None + context_limit: int = DEFAULT_CONTEXT_LIMIT + detail: str = "standard" def validate(self) -> None: if not self.query.strip(): @@ -32,6 +35,9 @@ def validate(self) -> None: raise ValueError("Context budget must be zero or greater") if self.max_depth is not None and self.max_depth < 0: raise ValueError("Context max depth must be zero or greater") + if self.context_limit < 0: + raise ValueError("Context limit must be zero or greater") + _validate_detail(self.detail) if self.profile not in CONTEXT_PROFILES: valid = ", ".join(sorted(CONTEXT_PROFILES)) raise ValueError(f"Unknown context profile: {self.profile}. Valid profiles: {valid}") @@ -52,7 +58,21 @@ class SearchHit: context: list[ContextNode] = field(default_factory=list) index_order: int = 0 - def as_dict(self) -> dict[str, Any]: + def as_dict(self, *, detail: str = "standard") -> dict[str, Any]: + _validate_detail(detail) + if detail == "slim": + payload: dict[str, Any] = { + "id": self.id, + "type": self.type, + "label": self.label, + "rank_score": self.rank_score, + } + _set_non_empty(payload, "path", self.path) + _set_non_empty(payload, "span", dict(self.span)) + _set_meaningful_summary(payload, self.summary, self.label) + context = [node.as_dict(detail=detail) for node in self.context] + _set_non_empty(payload, "context", context) + return payload return { "id": self.id, "type": self.type, @@ -64,7 +84,7 @@ def as_dict(self) -> dict[str, Any]: "rank_score": self.rank_score, "score_components": dict(self.score_components), "summary": self.summary, - "context": [node.as_dict() for node in self.context], + "context": [node.as_dict(detail=detail) for node in self.context], } @@ -76,13 +96,14 @@ class CompactContextPayload: budget: int results: tuple[SearchHit, ...] - def as_dict(self) -> dict[str, Any]: + def as_dict(self, *, detail: str = "standard") -> dict[str, Any]: + _validate_detail(detail) return { "query": self.query, "profile": self.profile, "limit": self.limit, "budget": self.budget, - "results": [hit.as_dict() for hit in self.results], + "results": [hit.as_dict(detail=detail) for hit in self.results], } @@ -114,7 +135,7 @@ def search(self, request: SearchRequest) -> CompactContextPayload: hit.id, hit.type, profile=request.profile, - limit=DEFAULT_CONTEXT_LIMIT, + limit=request.context_limit, budget=request.budget, max_depth=request.max_depth, ) @@ -301,8 +322,25 @@ def _span(line_start: Any, line_end: Any) -> dict[str, int]: return span +def _validate_detail(detail: str) -> None: + if detail not in DETAIL_LEVELS: + valid = ", ".join(sorted(DETAIL_LEVELS)) + raise ValueError(f"Unknown detail level: {detail}. Valid levels: {valid}") + + +def _set_non_empty(payload: dict[str, Any], key: str, value: Any) -> None: + if value not in ("", None, [], {}): + payload[key] = value + + +def _set_meaningful_summary(payload: dict[str, Any], summary: str, label: str) -> None: + if summary and summary != label: + payload["summary"] = summary + + __all__ = [ "CompactContextPayload", + "DETAIL_LEVELS", "DEFAULT_SEARCH_LIMIT", "FTSIndexSpec", "MAX_CANDIDATE_LIMIT", diff --git a/src/codebase_graph/setup/instructions.py b/src/codebase_graph/setup/instructions.py index 0cc7388..a62fadd 100644 --- a/src/codebase_graph/setup/instructions.py +++ b/src/codebase_graph/setup/instructions.py @@ -71,10 +71,10 @@ def _instruction_block(*, server_name: str, config_path: Path, setup_command: st f"{START_MARKER}\n" "## codebaseGraph workflow\n" "- Treat the repo-local `.codebaseGraph` graph as the project operating source of truth.\n" - f"- Use `{setup_command} graph-search --repo-root . --no-refresh --json` before answering repo-structure questions or performing coding tasks.\n" - f"- Use `{setup_command} graph-context --repo-root . --profile --no-refresh --json` when relationships or nearby evidence matter; useful profiles include `definitions`, `dependencies`, `callgraph`, `docs`, `runtime`, and `change_impact`.\n" + f"- Use `{setup_command} graph-search --repo-root . --no-refresh --detail slim --context-limit 1 --json` before answering repo-structure questions or performing coding tasks.\n" + f"- Use `{setup_command} graph-context --repo-root . --profile --no-refresh --detail slim --context-limit 2 --json` when relationships or nearby evidence matter; useful profiles include `definitions`, `dependencies`, `callgraph`, `docs`, `runtime`, and `change_impact`.\n" f"- For architecture orientation, run `{setup_command} graph-architecture-queries`, then execute selected read-only statements with `{setup_command} graph-query \"\" --repo-root .`.\n" - f"- Use `{setup_command} graph-schema` or `{setup_command} graph-query-helpers` before writing raw graph queries, and keep `graph-query` read-only.\n" + f"- Use `{setup_command} graph-schema` or `{setup_command} graph-query-helpers` before writing raw graph queries, add `--pretty` for indented JSON when humans need to inspect output, and keep `graph-query` read-only.\n" f"- Refresh the graph with `{setup_command} setup --repo-root . --mcp-client none` when files change materially. Setup config: `{config_path.as_posix()}`.\n" f"{END_MARKER}\n" ) diff --git a/tests/test_mcp_portability.py b/tests/test_mcp_portability.py index c690a98..c103acf 100644 --- a/tests/test_mcp_portability.py +++ b/tests/test_mcp_portability.py @@ -181,8 +181,12 @@ def test_stdio_mcp_wire_initialize_list_call_and_tool_error(tmp_path: Path) -> N assert initialized["result"]["protocolVersion"] == "2025-11-25" assert {tool["name"] for tool in listed["result"]["tools"]} >= {"graph_health", "graph_search", "graph_query"} + graph_search_tool = next(tool for tool in listed["result"]["tools"] if tool["name"] == "graph_search") + assert "context_limit" in graph_search_tool["inputSchema"]["properties"] + assert graph_search_tool["inputSchema"]["properties"]["detail"]["enum"] == ["slim", "standard"] assert health["result"]["structuredContent"]["ok"] is True assert search["result"]["structuredContent"]["results"] + assert "\n " not in search["result"]["content"][0]["text"] assert "error" not in failure assert failure["result"]["isError"] is True assert failure["result"]["structuredContent"]["error"]["type"] == "ValueError" diff --git a/tests/test_search.py b/tests/test_search.py index 685998d..859cbe0 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -12,8 +12,8 @@ from codebase_graph.ingest import GraphMaterializer from codebase_graph.mcp.runtime import GraphRuntimeConfig from codebase_graph.mcp.tools import handle_tool_call -from codebase_graph.reasoning import CompactContextBuilder -from codebase_graph.retrieval.search import SearchHit, SearchRequest, SearchService +from codebase_graph.reasoning import CompactContextBuilder, ContextNode +from codebase_graph.retrieval.search import CompactContextPayload, SearchHit, SearchRequest, SearchService class _Result: @@ -243,11 +243,84 @@ def test_search_service_uses_query_adapter_for_fts() -> None: assert adapter.search_calls +def test_search_request_rejects_invalid_context_limit_and_detail() -> None: + with pytest.raises(ValueError, match="Context limit must be zero or greater"): + SearchRequest("SampleService", context_limit=-1).validate() + with pytest.raises(ValueError, match="Unknown detail level"): + SearchRequest("SampleService", detail="debug").validate() + + +def test_search_service_respects_zero_context_limit() -> None: + adapter = _Adapter() + + payload = SearchService(_AdapterStore(adapter)).search(SearchRequest("SampleService", limit=1, context_limit=0)) + + data = payload.as_dict() + assert data["results"][0]["context"] == [] + assert adapter.search_calls + assert adapter.neighbor_calls == [] + + def test_search_request_rejects_invalid_profile() -> None: with pytest.raises(ValueError, match="Unknown context profile"): SearchRequest("SampleService", profile="missing").validate() +def test_slim_payload_omits_diagnostics_and_duplicate_summaries() -> None: + payload = CompactContextPayload( + query="run", + profile="brief", + limit=1, + budget=600, + results=( + SearchHit( + id="Method:run", + type="Method", + label="run", + qualified_name="sample.Service.run", + path="sample/service.py", + span={"line_start": 4, "line_end": 8}, + score=2.0, + rank_score=0.9, + score_components={"fts": 1.0}, + summary="run", + context=[ + ContextNode("Defines", "incoming", "Module", "sample.service", "sample/service.py", summary="sample.service"), + ContextNode("Documents", "outgoing", "DocumentationChunk", "Usage", "README.md", summary="Use run to start the service."), + ], + ), + ), + ) + + hit = payload.as_dict(detail="slim")["results"][0] + + assert hit == { + "id": "Method:run", + "type": "Method", + "label": "run", + "rank_score": 0.9, + "path": "sample/service.py", + "span": {"line_start": 4, "line_end": 8}, + "context": [ + { + "relation": "Defines", + "direction": "incoming", + "type": "Module", + "label": "sample.service", + "path": "sample/service.py", + }, + { + "relation": "Documents", + "direction": "outgoing", + "type": "DocumentationChunk", + "label": "Usage", + "path": "README.md", + "summary": "Use run to start the service.", + }, + ], + } + + def test_search_service_returns_sample_class_with_compact_context(tmp_path: Path) -> None: _require_graph_runtime() materializer = _materialize_fixture(tmp_path, include_fts=True) @@ -394,7 +467,14 @@ def test_cli_graph_commands_match_mcp_tool_payloads(tmp_path: Path, capsys: pyte ]) == 0 assert json.loads(capsys.readouterr().out) == handle_tool_call("graph_health", {}, runtime=runtime) - search_args = {"query": "SampleService", "limit": 2, "profile": "brief", "budget": 600} + search_args = { + "query": "SampleService", + "limit": 2, + "profile": "brief", + "budget": 600, + "context_limit": 1, + "detail": "slim", + } assert cli_main([ "graph-search", "SampleService", @@ -406,11 +486,17 @@ def test_cli_graph_commands_match_mcp_tool_payloads(tmp_path: Path, capsys: pyte manifest_path.as_posix(), "--limit", "2", + "--context-limit", + "1", + "--detail", + "slim", "--no-refresh", "--json", ]) == 0 search_payload = json.loads(capsys.readouterr().out) assert search_payload == handle_tool_call("graph_search", search_args, runtime=runtime) + assert "score" not in search_payload["results"][0] + assert len(search_payload["results"][0].get("context", [])) <= 1 hit = next(item for item in search_payload["results"] if item["label"] == "SampleService") context_args = { @@ -419,6 +505,8 @@ def test_cli_graph_commands_match_mcp_tool_payloads(tmp_path: Path, capsys: pyte "limit": 1, "profile": "definitions", "budget": 600, + "context_limit": 3, + "detail": "slim", } assert cli_main([ "graph-context", @@ -436,6 +524,8 @@ def test_cli_graph_commands_match_mcp_tool_payloads(tmp_path: Path, capsys: pyte "definitions", "--limit", "1", + "--detail", + "slim", ]) == 0 assert json.loads(capsys.readouterr().out) == handle_tool_call("graph_context", context_args, runtime=runtime) @@ -458,10 +548,17 @@ def test_cli_graph_commands_match_mcp_tool_payloads(tmp_path: Path, capsys: pyte def test_cli_graph_metadata_commands_do_not_open_graph_db(capsys: pytest.CaptureFixture[str]) -> None: assert cli_main(["graph-schema"]) == 0 - schema = json.loads(capsys.readouterr().out) + schema_output = capsys.readouterr().out + assert "\n " not in schema_output + schema = json.loads(schema_output) assert schema["ontology"] assert schema["context_profiles"] + assert cli_main(["graph-schema", "--pretty"]) == 0 + pretty_schema_output = capsys.readouterr().out + assert "\n " in pretty_schema_output + assert json.loads(pretty_schema_output)["ontology"] + assert cli_main(["graph-query-helpers"]) == 0 helpers = json.loads(capsys.readouterr().out) assert any(helper["name"] == "repository_overview" for helper in helpers["query_helpers"]) From ac790812b196723e817b5acd14c97628b02b081a Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Wed, 27 May 2026 12:46:10 +0930 Subject: [PATCH 24/53] docs: enhance instructions --- AGENTS.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index d17471c..ba6eb95 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,12 +1,12 @@ ## codebaseGraph workflow - Treat the repo-local `.codebaseGraph` graph as the project operating source of truth. -- Use `/Users/rabii/Projects/Repositories/codebaseGraph/.venv/bin/codebase-graph graph-search --repo-root . --no-refresh --json` before answering repo-structure questions or performing coding tasks. -- Use `/Users/rabii/Projects/Repositories/codebaseGraph/.venv/bin/codebase-graph graph-context --repo-root . --profile --no-refresh --json` when relationships or nearby evidence matter; useful profiles include `definitions`, `dependencies`, `callgraph`, `docs`, `runtime`, and `change_impact`. +- Use `/Users/rabii/Projects/Repositories/codebaseGraph/.venv/bin/codebase-graph graph-search --repo-root . --no-refresh --detail slim --context-limit 1 --json` before answering repo-structure questions or performing coding tasks. +- Use `/Users/rabii/Projects/Repositories/codebaseGraph/.venv/bin/codebase-graph graph-context --repo-root . --profile --no-refresh --detail slim --context-limit 2 --json` when relationships or nearby evidence matter; useful profiles include `definitions`, `dependencies`, `callgraph`, `docs`, `runtime`, and `change_impact`. - For architecture orientation, run `/Users/rabii/Projects/Repositories/codebaseGraph/.venv/bin/codebase-graph graph-architecture-queries`, then execute selected read-only statements with `/Users/rabii/Projects/Repositories/codebaseGraph/.venv/bin/codebase-graph graph-query "" --repo-root .`. -- Use `/Users/rabii/Projects/Repositories/codebaseGraph/.venv/bin/codebase-graph graph-schema` or `/Users/rabii/Projects/Repositories/codebaseGraph/.venv/bin/codebase-graph graph-query-helpers` before writing raw graph queries, and keep `graph-query` read-only. +- Use `/Users/rabii/Projects/Repositories/codebaseGraph/.venv/bin/codebase-graph graph-schema` or `/Users/rabii/Projects/Repositories/codebaseGraph/.venv/bin/codebase-graph graph-query-helpers` before writing raw graph queries, add `--pretty` for indented JSON when humans need to inspect output, and keep `graph-query` read-only. - Refresh the graph with `/Users/rabii/Projects/Repositories/codebaseGraph/.venv/bin/codebase-graph setup --repo-root . --mcp-client none` when files change materially. Setup config: `/Users/rabii/Projects/Repositories/codebaseGraph/.codebaseGraph/config.json`. ## Git Commit Convention -- Strictly use Conventional Commits 1.0.0 for commit message. +- When you finish your coding task, strictly use Conventional Commits 1.0.0 for commit message and commit your changes. From a534c672211c575581e4e9b6529b6985bbda372d Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Wed, 27 May 2026 13:47:09 +0930 Subject: [PATCH 25/53] fix: harden MCP runtime for production readiness Bound graph_query responses at the cursor boundary, return JSON-RPC parse errors for malformed stdio frames, enforce local-only HTTP binding by default, and add a single-writer lock around on-disk graph materialization. Strengthen health checks and release gates so packaged artifacts exercise setup, graph search, and MCP handshake behavior before publishing. Constraint: MCP remains a local-first CLI/server surface, so remote HTTP binding is explicit and still unauthenticated Rejected: Keep graph_query get_all with response slicing | it still permits unbounded memory pressure Rejected: Treat malformed stdio frames as process-fatal | MCP clients need protocol errors instead of server exits Confidence: high Scope-risk: moderate Directive: Do not relax graph_query or HTTP bounds without replacing them with equivalent resource controls Tested: ./.venv/bin/python -m pytest Tested: ./.venv/bin/ruff check . Tested: ./.venv/bin/python scripts/smoke_built_wheel.py ./.venv/bin/codebase-graph Tested: ./.venv/bin/python -m build --no-isolation --outdir /private/tmp/codebasegraph-dist-review-staged Tested: ./.venv/bin/python -m twine check /private/tmp/codebasegraph-dist-review-staged/* Tested: runtime checks for graph-health, bounded graph-query, malformed stdio, YAML syntax, pip check, and graph refresh Not-tested: GitHub-hosted CI matrix and PyPI Trusted Publishing execution --- .github/dependabot.yml | 12 ++ .github/workflows/ci.yml | 1 + .github/workflows/release.yml | 3 +- README.md | 10 +- docs/release.md | 3 +- scripts/smoke_built_wheel.py | 126 +++++++++++++++++++++ src/codebase_graph/cli/__init__.py | 6 + src/codebase_graph/ingest/materializer.py | 103 +++++++++++------ src/codebase_graph/mcp/tools.py | 46 ++++++-- src/codebase_graph/mcp/transports/http.py | 28 ++++- src/codebase_graph/mcp/transports/stdio.py | 37 +++++- tests/test_materializer.py | 18 +++ tests/test_mcp_portability.py | 47 ++++++++ tests/test_search.py | 38 ++++++- tests/test_setup_workflow.py | 4 + 15 files changed, 427 insertions(+), 55 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 scripts/smoke_built_wheel.py diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..d3c3021 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,12 @@ +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" + open-pull-requests-limit: 5 + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + open-pull-requests-limit: 5 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index adf2d0a..f78eafa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -106,3 +106,4 @@ jobs: /tmp/codebase-graph-wheel/bin/python -m pip install dist/*.whl /tmp/codebase-graph-wheel/bin/codebase-graph --help /tmp/codebase-graph-wheel/bin/codebase-graph-mcp --help + /tmp/codebase-graph-wheel/bin/python scripts/smoke_built_wheel.py /tmp/codebase-graph-wheel/bin/codebase-graph diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5fe6f92..871d09d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -110,6 +110,7 @@ jobs: /tmp/codebase-graph-wheel/bin/python -m pip install dist/*.whl /tmp/codebase-graph-wheel/bin/codebase-graph --help /tmp/codebase-graph-wheel/bin/codebase-graph-mcp --help + /tmp/codebase-graph-wheel/bin/python scripts/smoke_built_wheel.py /tmp/codebase-graph-wheel/bin/codebase-graph - name: Upload distributions uses: actions/upload-artifact@v4 @@ -132,7 +133,7 @@ jobs: runs-on: ubuntu-latest environment: name: pypi - url: https://pypi.org/p/cbasegraph + url: https://pypi.org/p/codebase-graph permissions: id-token: write diff --git a/README.md b/README.md index 5aa8d3b..5c7db19 100644 --- a/README.md +++ b/README.md @@ -144,7 +144,8 @@ Stdio is the default transport for local MCP clients. An optional local Streamab codebase-graph mcp http --config .codebaseGraph/config.json --host 127.0.0.1 --port 8765 ``` -Keep HTTP bound to `127.0.0.1` unless you have added authentication and understand the DNS rebinding risk for local MCP servers. +The HTTP transport rejects non-local bind hosts unless `--allow-remote` is passed. Keep it bound to `127.0.0.1` +for normal use; `--allow-remote` does not add authentication, TLS, rate limiting, or a multi-user session model. Available MCP tools: @@ -156,6 +157,9 @@ Available MCP tools: - `graph_architecture_queries` - `graph_query` with write-like statements blocked +`graph_query` returns at most 1,000 rows per call and fetches only one extra row to determine whether the result was +truncated. Add a narrower `MATCH` pattern or a query-side `LIMIT` for broader graph exploration. + For coding-task architecture orientation, call `graph_architecture_queries` first to fetch the grouped read-only Cypher catalog, then run selected statements with `graph_query`. ## CLI graph workflow @@ -198,7 +202,7 @@ ruff check . ## CI and releases -GitHub Actions runs pytest across Linux, macOS, and Windows for Python 3.10 through 3.14, plus ruff and package-build validation. Releases are managed by release-please, use tag-derived package versions, create GitHub Releases with distribution assets, and publish to PyPI through Trusted Publishing. +GitHub Actions runs pytest across Linux, macOS, and Windows for Python 3.10 through 3.14, plus ruff and package-build validation. Built wheels are smoke-tested with `setup`, `graph-health`, `graph-search`, and a stdio MCP handshake before release. Releases are managed by release-please, use tag-derived package versions, create GitHub Releases with distribution assets, and publish to PyPI through Trusted Publishing. Conda distribution uses the conda-forge staged-recipes path rather than direct Anaconda.org uploads. See [docs/release.md](docs/release.md) for the release workflow and conda-forge submission checklist. @@ -212,4 +216,4 @@ Conda distribution uses the conda-forge staged-recipes path rather than direct A - PATH or executable issues: run setup from the virtual environment that contains `codebase-graph`; the descriptor prefers that absolute executable path. - Direct smoke test: run `codebase-graph mcp serve --config .codebaseGraph/config.json` and send MCP `initialize`, `tools/list`, and `tools/call` JSON-RPC messages over stdio. - Unsupported files: binary, vendor, cache, virtualenv, build, dist, `.codebase_graph`, and `.codebaseGraph` paths are skipped. -- Lock/contention errors: stop other graph materialization or MCP processes using the same `.codebaseGraph/_graph.ldb`, then rerun setup. +- Lock/contention errors: stop other graph materialization or setup processes using the same `.codebaseGraph/_graph.ldb`. If no writer is running, remove the stale `.ldb.lock` file named in the error, then rerun setup. diff --git a/docs/release.md b/docs/release.md index 72c6668..b9b7727 100644 --- a/docs/release.md +++ b/docs/release.md @@ -19,7 +19,8 @@ Pull requests and pushes to `main` or `codex/**` run: - `pytest` on Linux, macOS, and Windows for Python 3.10 through 3.14. - `ruff check .` on Linux. -- A package build on Linux with `python -m build`, `twine check`, and console-script smoke tests from the built wheel. +- A package build on Linux with `python -m build`, `twine check`, console-script smoke tests from the built wheel, + and a packaged runtime smoke that runs `setup`, `graph-health`, `graph-search`, and stdio MCP handshake checks. ## Release flow diff --git a/scripts/smoke_built_wheel.py b/scripts/smoke_built_wheel.py new file mode 100644 index 0000000..27a60d8 --- /dev/null +++ b/scripts/smoke_built_wheel.py @@ -0,0 +1,126 @@ +from __future__ import annotations + +import json +import subprocess +import sys +import tempfile +from pathlib import Path +from typing import Any, BinaryIO + + +def main(argv: list[str]) -> int: + if len(argv) != 2: + raise SystemExit("usage: smoke_built_wheel.py /path/to/codebase-graph") + executable = Path(argv[1]) + with tempfile.TemporaryDirectory(prefix="codebase-graph-wheel-smoke-") as tmp_dir: + repo_root = _sample_repo(Path(tmp_dir) / "sample_repo") + setup = _run( + [ + executable.as_posix(), + "setup", + "--repo-root", + repo_root.as_posix(), + "--mcp-client", + "none", + "--instructions-target", + "skip", + ] + ) + setup_payload = json.loads(setup.stdout) + config_path = Path(setup_payload["config_path"]) + + health = json.loads(_run([executable.as_posix(), "graph-health", "--repo-root", repo_root.as_posix()]).stdout) + if not health.get("ok") or not health.get("graph_readable"): + raise AssertionError(f"graph-health failed readiness smoke: {health}") + + search = json.loads( + _run( + [ + executable.as_posix(), + "graph-search", + "SampleService", + "--repo-root", + repo_root.as_posix(), + "--no-refresh", + "--detail", + "slim", + "--json", + ] + ).stdout + ) + if not search.get("results"): + raise AssertionError(f"graph-search returned no results: {search}") + + _mcp_smoke([executable.as_posix(), "mcp", "serve", "--config", config_path.as_posix()]) + return 0 + + +def _run(command: list[str]) -> subprocess.CompletedProcess[str]: + return subprocess.run(command, capture_output=True, text=True, check=True) + + +def _sample_repo(repo_root: Path) -> Path: + package = repo_root / "sample_project" + package.mkdir(parents=True) + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "service.py").write_text( + "class SampleService:\n" + " def run(self) -> str:\n" + " return helper()\n\n" + "def helper() -> str:\n" + " return 'ok'\n", + encoding="utf-8", + ) + (repo_root / "README.md").write_text("# Sample Repo\n\nSampleService smoke fixture.\n", encoding="utf-8") + return repo_root + + +def _mcp_smoke(command: list[str]) -> None: + proc = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + assert proc.stdin is not None + assert proc.stdout is not None + try: + initialized = _rpc(proc.stdin, proc.stdout, "initialize", {"protocolVersion": "2025-11-25"}) + listed = _rpc(proc.stdin, proc.stdout, "tools/list", {}) + health = _rpc(proc.stdin, proc.stdout, "tools/call", {"name": "graph_health", "arguments": {}}) + finally: + proc.stdin.close() + proc.wait(timeout=10) + assert proc.stderr is not None + stderr = proc.stderr.read() + if proc.returncode != 0: + raise AssertionError(stderr.decode("utf-8", errors="replace")) + if initialized["result"]["protocolVersion"] != "2025-11-25": + raise AssertionError(initialized) + tool_names = {tool["name"] for tool in listed["result"]["tools"]} + if not {"graph_health", "graph_search", "graph_query"}.issubset(tool_names): + raise AssertionError(listed) + if health["result"]["structuredContent"].get("ok") is not True: + raise AssertionError(health) + + +def _rpc(stdin: BinaryIO, stdout: BinaryIO, method: str, params: dict[str, Any]) -> dict[str, Any]: + request_id = _rpc.counter + _rpc.counter += 1 + body = json.dumps({"jsonrpc": "2.0", "id": request_id, "method": method, "params": params}).encode("utf-8") + stdin.write(f"Content-Length: {len(body)}\r\n\r\n".encode("ascii") + body) + stdin.flush() + return _read_response(stdout) + + +_rpc.counter = 1 # type: ignore[attr-defined] + + +def _read_response(stdout: BinaryIO) -> dict[str, Any]: + header = stdout.readline() + if not header.lower().startswith(b"content-length:"): + raise AssertionError(f"unexpected MCP header: {header!r}") + length = int(header.split(b":", 1)[1].strip()) + separator = stdout.readline() + if separator not in {b"\r\n", b"\n"}: + raise AssertionError(f"unexpected MCP header separator: {separator!r}") + return json.loads(stdout.read(length).decode("utf-8")) + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) diff --git a/src/codebase_graph/cli/__init__.py b/src/codebase_graph/cli/__init__.py index 2669243..c138e83 100644 --- a/src/codebase_graph/cli/__init__.py +++ b/src/codebase_graph/cli/__init__.py @@ -114,6 +114,11 @@ def main(argv: Sequence[str] | None = None) -> int: http_parser.add_argument("--host", default="127.0.0.1", help="HTTP bind host; default keeps the server local") http_parser.add_argument("--port", type=int, default=8765, help="HTTP bind port") http_parser.add_argument("--path", default="/mcp", help="MCP HTTP endpoint path") + http_parser.add_argument( + "--allow-remote", + action="store_true", + help="Allow binding MCP HTTP to a non-local host; no authentication is provided", + ) args = parser.parse_args(argv) if args.command == "materialize": @@ -262,6 +267,7 @@ def main(argv: Sequence[str] | None = None) -> int: host=args.host, port=args.port, endpoint_path=args.path, + allow_remote=args.allow_remote, ) return 0 parser.error(f"Unknown command: {args.command}") diff --git a/src/codebase_graph/ingest/materializer.py b/src/codebase_graph/ingest/materializer.py index f49946c..30f8075 100644 --- a/src/codebase_graph/ingest/materializer.py +++ b/src/codebase_graph/ingest/materializer.py @@ -353,44 +353,48 @@ def _materialize_full_atomic( supported: Mapping[str, SourceSnapshot], diff: ManifestDiff, ) -> MaterializationResult: - rebuilt_entries: dict[str, ManifestEntry] = {} - rebuilt_graphs: dict[str, CodeGraph] = {} - for path in diff.rebuild_paths: - snapshot = supported[path] - graph = self._build_graph(snapshot) - rebuilt_graphs[path] = graph - rebuilt_entries[path] = _manifest_entry(snapshot, graph) - - next_manifest = MaterializationManifest(parser_version=self.parser_version, files=rebuilt_entries) target_db_path = _filesystem_db_path(self.db_path) - temp_db_path = _temporary_sibling(target_db_path, suffix=".lbug.tmp") - temp_manifest_path = _temporary_sibling(self.manifest_path, suffix=".manifest.tmp") - marker_path = self._rebuild_marker_path - temp_store: LadybugCodeGraphStore | None = None + lock_fd, lock_path = _acquire_materialization_lock(target_db_path) try: - temp_store = create_ladybug_database(temp_db_path, include_fts=self.include_fts) - if rebuilt_graphs: - temp_store.insert_graphs_bulk([rebuilt_graphs[path] for path in sorted(rebuilt_graphs)]) - temp_store.close() - temp_store = None - - next_manifest.write(temp_manifest_path) - _write_rebuild_marker(marker_path, target_db_path, self.manifest_path) - self._close_store() - _unlink_db_sidecars(target_db_path) - os.replace(temp_db_path, target_db_path) - os.replace(temp_manifest_path, self.manifest_path) - _unlink_db_sidecars(target_db_path) - _unlink_if_exists(marker_path) - self._store = None - except Exception: - if temp_store is not None: + rebuilt_entries: dict[str, ManifestEntry] = {} + rebuilt_graphs: dict[str, CodeGraph] = {} + for path in diff.rebuild_paths: + snapshot = supported[path] + graph = self._build_graph(snapshot) + rebuilt_graphs[path] = graph + rebuilt_entries[path] = _manifest_entry(snapshot, graph) + + next_manifest = MaterializationManifest(parser_version=self.parser_version, files=rebuilt_entries) + temp_db_path = _temporary_sibling(target_db_path, suffix=".lbug.tmp") + temp_manifest_path = _temporary_sibling(self.manifest_path, suffix=".manifest.tmp") + marker_path = self._rebuild_marker_path + temp_store: LadybugCodeGraphStore | None = None + try: + temp_store = create_ladybug_database(temp_db_path, include_fts=self.include_fts) + if rebuilt_graphs: + temp_store.insert_graphs_bulk([rebuilt_graphs[path] for path in sorted(rebuilt_graphs)]) temp_store.close() - _unlink_if_exists(temp_db_path) - _unlink_db_sidecars(temp_db_path) - _unlink_if_exists(temp_manifest_path) - _unlink_if_exists(temp_manifest_path.with_suffix(temp_manifest_path.suffix + ".tmp")) - raise + temp_store = None + + next_manifest.write(temp_manifest_path) + _write_rebuild_marker(marker_path, target_db_path, self.manifest_path) + self._close_store() + _unlink_db_sidecars(target_db_path) + os.replace(temp_db_path, target_db_path) + os.replace(temp_manifest_path, self.manifest_path) + _unlink_db_sidecars(target_db_path) + _unlink_if_exists(marker_path) + self._store = None + except Exception: + if temp_store is not None: + temp_store.close() + _unlink_if_exists(temp_db_path) + _unlink_db_sidecars(temp_db_path) + _unlink_if_exists(temp_manifest_path) + _unlink_if_exists(temp_manifest_path.with_suffix(temp_manifest_path.suffix + ".tmp")) + raise + finally: + _release_materialization_lock(lock_fd, lock_path) return _materialization_result( mode=mode, @@ -499,6 +503,35 @@ def _temporary_sibling(path: Path, *, suffix: str) -> Path: return Path(temp_path) +def _acquire_materialization_lock(db_path: Path) -> tuple[int, Path]: + lock_path = Path(f"{db_path}.lock") + lock_path.parent.mkdir(parents=True, exist_ok=True) + try: + descriptor = os.open(lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY) + except FileExistsError as exc: + raise RuntimeError( + f"codebaseGraph materialization is already in progress for {db_path}. " + f"If no materializer is running, remove the stale lock file: {lock_path}" + ) from exc + payload = { + "created_at": datetime.now(timezone.utc).isoformat(), + "pid": os.getpid(), + "db_path": db_path.as_posix(), + } + try: + os.write(descriptor, (json.dumps(payload, sort_keys=True) + "\n").encode("utf-8")) + except Exception: + os.close(descriptor) + _unlink_if_exists(lock_path) + raise + return descriptor, lock_path + + +def _release_materialization_lock(descriptor: int, lock_path: Path) -> None: + os.close(descriptor) + _unlink_if_exists(lock_path) + + def _write_rebuild_marker(marker_path: Path, db_path: Path, manifest_path: Path) -> None: marker_path.parent.mkdir(parents=True, exist_ok=True) tmp_path = marker_path.with_suffix(marker_path.suffix + ".tmp") diff --git a/src/codebase_graph/mcp/tools.py b/src/codebase_graph/mcp/tools.py index 43c7950..76f9df4 100644 --- a/src/codebase_graph/mcp/tools.py +++ b/src/codebase_graph/mcp/tools.py @@ -15,6 +15,7 @@ r"\b(CREATE|DELETE|SET|MERGE|DROP|COPY|INSERT|LOAD|INSTALL|DETACH|REMOVE|ALTER|RENAME)\b", re.IGNORECASE, ) +MAX_GRAPH_QUERY_LIMIT = 1000 class UnknownToolError(ValueError): @@ -125,7 +126,7 @@ def tool_specs() -> list[dict[str, Any]]: "properties": { "statement": {"type": "string"}, "parameters": {"type": "object"}, - "limit": {"type": "integer", "minimum": 1}, + "limit": {"type": "integer", "minimum": 1, "maximum": MAX_GRAPH_QUERY_LIMIT}, }, "required": ["statement"], "additionalProperties": False, @@ -135,12 +136,27 @@ def tool_specs() -> list[dict[str, Any]]: def _health(runtime: GraphRuntimeConfig) -> dict[str, Any]: - return { - "ok": runtime.db_path.exists(), + payload: dict[str, Any] = { + "ok": False, "repo_root": runtime.repo_root.as_posix(), "database_path": runtime.db_path.as_posix(), "manifest_path": runtime.manifest_path.as_posix() if runtime.manifest_path else None, + "database_exists": runtime.db_path.exists(), + "manifest_exists": runtime.manifest_path.exists() if runtime.manifest_path else None, } + if not runtime.db_path.exists(): + return payload + try: + with open_graph_store(runtime) as store: + rows = store.execute("MATCH (n) RETURN count(n) AS total_nodes LIMIT 1").get_n(1) + except Exception as exc: + payload["graph_readable"] = False + payload["error"] = {"type": exc.__class__.__name__, "message": str(exc)} + return payload + payload["ok"] = True + payload["graph_readable"] = True + payload["total_nodes"] = _json_safe(rows[0][0]) if rows and rows[0] else 0 + return payload def _search_request(arguments: dict[str, Any]) -> SearchRequest: @@ -189,12 +205,19 @@ def _query_payload(store: LadybugCodeGraphStore, arguments: dict[str, Any]) -> d parameters = arguments.get("parameters") or {} if not isinstance(parameters, dict): raise ValueError("graph_query parameters must be a JSON object") - limit = int(arguments.get("limit", 100)) - rows = store.execute(statement, parameters).get_all() + limit = _graph_query_limit(arguments) + result = store.execute(statement, parameters) + try: + rows = result.get_n(limit + 1) + finally: + close = getattr(result, "close", None) + if callable(close): + close() + visible_rows = rows[:limit] return { "statement": statement, - "row_count": len(rows), - "rows": [_row_values(row) for row in rows[:limit]], + "row_count": len(visible_rows), + "rows": [_row_values(row) for row in visible_rows], "truncated": len(rows) > limit, } @@ -208,6 +231,15 @@ def _validate_read_only_statement(statement: str) -> None: raise ValueError(f"graph_query is read-only; blocked keyword: {match.group(1).upper()}") +def _graph_query_limit(arguments: dict[str, Any]) -> int: + limit = int(arguments.get("limit", 100)) + if limit <= 0: + raise ValueError("graph_query limit must be greater than zero") + if limit > MAX_GRAPH_QUERY_LIMIT: + raise ValueError(f"graph_query limit must be {MAX_GRAPH_QUERY_LIMIT} or less") + return limit + + def _row_values(row: Any) -> list[Any]: try: return [_json_safe(value) for value in row] diff --git a/src/codebase_graph/mcp/transports/http.py b/src/codebase_graph/mcp/transports/http.py index 7e74bad..200f628 100644 --- a/src/codebase_graph/mcp/transports/http.py +++ b/src/codebase_graph/mcp/transports/http.py @@ -10,6 +10,7 @@ from codebase_graph.mcp.protocol import SUPPORTED_PROTOCOL_VERSIONS, McpGraphServer, rpc_error LOCAL_ORIGINS = {"localhost", "127.0.0.1", "::1"} +MAX_HTTP_BODY_BYTES = 1_000_000 class McpHttpServer(ThreadingHTTPServer): @@ -28,7 +29,10 @@ def build_http_server( host: str = "127.0.0.1", port: int = 8765, endpoint_path: str = "/mcp", + allow_remote: bool = False, ) -> McpHttpServer: + if not allow_remote and host not in LOCAL_ORIGINS: + raise ValueError("MCP HTTP transport may only bind to localhost unless allow_remote is enabled") graph_server = McpGraphServer.from_paths( repo_root=repo_root, config_path=config_path, @@ -50,6 +54,7 @@ def serve_http( host: str = "127.0.0.1", port: int = 8765, endpoint_path: str = "/mcp", + allow_remote: bool = False, ) -> None: server = build_http_server( repo_root=repo_root, @@ -59,6 +64,7 @@ def serve_http( host=host, port=port, endpoint_path=endpoint_path, + allow_remote=allow_remote, ) try: server.serve_forever() @@ -74,8 +80,10 @@ def do_POST(self) -> None: return if not self._valid_protocol_header(): return + length = self._content_length() + if length is None: + return try: - length = int(self.headers.get("Content-Length", "0")) message = json.loads(self.rfile.read(length).decode("utf-8")) except Exception as exc: self._send_json(rpc_error(None, -32700, f"Invalid JSON-RPC payload: {exc}"), status=HTTPStatus.BAD_REQUEST) @@ -133,6 +141,24 @@ def _valid_protocol_header(self) -> bool: ) return False + def _content_length(self) -> int | None: + raw_length = self.headers.get("Content-Length", "0") + try: + length = int(raw_length) + except ValueError: + self._send_json(rpc_error(None, -32600, "Content-Length must be an integer"), status=HTTPStatus.BAD_REQUEST) + return None + if length < 0: + self._send_json(rpc_error(None, -32600, "Content-Length must be non-negative"), status=HTTPStatus.BAD_REQUEST) + return None + if length > MAX_HTTP_BODY_BYTES: + self._send_json( + rpc_error(None, -32000, "MCP request body is too large", {"max_bytes": MAX_HTTP_BODY_BYTES}), + status=HTTPStatus.REQUEST_ENTITY_TOO_LARGE, + ) + return None + return length + def _send_json(self, payload: dict[str, Any], *, status: HTTPStatus = HTTPStatus.OK) -> None: body = json.dumps(payload, separators=(",", ":"), sort_keys=True).encode("utf-8") self.send_response(status) diff --git a/src/codebase_graph/mcp/transports/stdio.py b/src/codebase_graph/mcp/transports/stdio.py index 23e480b..8a063d3 100644 --- a/src/codebase_graph/mcp/transports/stdio.py +++ b/src/codebase_graph/mcp/transports/stdio.py @@ -5,7 +5,11 @@ from pathlib import Path from typing import Any, BinaryIO -from codebase_graph.mcp.protocol import McpGraphServer +from codebase_graph.mcp.protocol import McpGraphServer, rpc_error + + +class StdioMessageError(ValueError): + pass def serve_stdio( @@ -22,7 +26,11 @@ def serve_stdio( manifest_path=manifest_path, ) while True: - message = read_message(sys.stdin.buffer) + try: + message = read_message(sys.stdin.buffer) + except StdioMessageError as exc: + write_message(sys.stdout.buffer, rpc_error(None, -32700, f"Invalid JSON-RPC payload: {exc}")) + continue if message is None: return response = server.handle_json_rpc(message) @@ -35,14 +43,21 @@ def read_message(stream: BinaryIO) -> dict[str, Any] | None: if not line: return None if line.lower().startswith(b"content-length:"): - length = int(line.split(b":", 1)[1].strip()) + try: + length = int(line.split(b":", 1)[1].strip()) + except ValueError as exc: + raise StdioMessageError("Content-Length must be an integer") from exc + if length < 0: + raise StdioMessageError("Content-Length must be non-negative") while True: header = stream.readline() if header in {b"\r\n", b"\n", b""}: break body = stream.read(length) - return json.loads(body.decode("utf-8")) - return json.loads(line.decode("utf-8")) + if len(body) != length: + raise StdioMessageError("Body ended before Content-Length bytes were read") + return _json_rpc_payload(body) + return _json_rpc_payload(line) def write_message(stream: BinaryIO, message: dict[str, Any]) -> None: @@ -50,3 +65,15 @@ def write_message(stream: BinaryIO, message: dict[str, Any]) -> None: stream.write(f"Content-Length: {len(body)}\r\n\r\n".encode("ascii")) stream.write(body) stream.flush() + + +def _json_rpc_payload(data: bytes) -> dict[str, Any]: + try: + payload = json.loads(data.decode("utf-8")) + except UnicodeDecodeError as exc: + raise StdioMessageError(f"Body must be UTF-8: {exc}") from exc + except json.JSONDecodeError as exc: + raise StdioMessageError(str(exc)) from exc + if not isinstance(payload, dict): + raise StdioMessageError("JSON-RPC payload must be an object") + return payload diff --git a/tests/test_materializer.py b/tests/test_materializer.py index b253454..851e4e6 100644 --- a/tests/test_materializer.py +++ b/tests/test_materializer.py @@ -407,6 +407,24 @@ def test_full_ondisk_materialization_replaces_stale_sidecars(tmp_path: Path) -> assert not Path(f"{db_path}.shadow").exists() +def test_ondisk_materialization_rejects_concurrent_writer_lock(tmp_path: Path) -> None: + pytest.importorskip("tree_sitter") + pytest.importorskip("tree_sitter_python") + pytest.importorskip("real_ladybug") + source_root = _copy_fixture(tmp_path) + db_path = tmp_path / "graph.lbug" + manifest_path = tmp_path / "manifest.json" + lock_path = Path(f"{db_path}.lock") + lock_path.write_text("{}\n", encoding="utf-8") + + with pytest.raises(RuntimeError, match="materialization is already in progress"): + GraphMaterializer(source_root, db_path=db_path, manifest_path=manifest_path, include_fts=False).materialize( + mode="full" + ) + + assert lock_path.exists() + + def test_pending_rebuild_marker_forces_changed_mode_atomic_rebuild( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/test_mcp_portability.py b/tests/test_mcp_portability.py index c103acf..2639584 100644 --- a/tests/test_mcp_portability.py +++ b/tests/test_mcp_portability.py @@ -192,6 +192,33 @@ def test_stdio_mcp_wire_initialize_list_call_and_tool_error(tmp_path: Path) -> N assert failure["result"]["structuredContent"]["error"]["type"] == "ValueError" +def test_stdio_mcp_malformed_frame_returns_parse_error(tmp_path: Path) -> None: + pytest.importorskip("tree_sitter") + pytest.importorskip("tree_sitter_python") + pytest.importorskip("real_ladybug") + repo_root = _fresh_repo(tmp_path) + result = run_setup(SetupOptions(repo_root=repo_root, mcp_client="none", instructions_target="skip")) + setup_payload = json.loads(result.paths.config_path.read_text(encoding="utf-8")) + + completed = subprocess.run( + setup_payload["mcp"]["command"], + input=b"Content-Length: 1\r\n\r\n{", + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + responses = _stdio_messages(completed.stdout) + assert completed.returncode == 0 + assert completed.stderr == b"" + assert responses[0]["error"]["code"] == -32700 + + +def test_http_mcp_rejects_remote_bind_without_explicit_opt_in(tmp_path: Path) -> None: + with pytest.raises(ValueError, match="localhost"): + build_http_server(repo_root=tmp_path, db_path=tmp_path / "missing.ldb", host="0.0.0.0", port=0) + + def test_http_mcp_transport_handles_initialize_list_and_call(tmp_path: Path) -> None: pytest.importorskip("tree_sitter") pytest.importorskip("tree_sitter_python") @@ -242,6 +269,26 @@ def _read_stdio_response(stdout: BinaryIO) -> dict[str, Any]: return json.loads(stdout.read(length).decode("utf-8")) +def _stdio_messages(data: bytes) -> list[dict[str, Any]]: + messages: list[dict[str, Any]] = [] + cursor = 0 + while cursor < len(data): + header_end = data.find(b"\r\n\r\n", cursor) + assert header_end != -1 + header = data[cursor:header_end].decode("ascii") + length = None + for line in header.splitlines(): + if line.lower().startswith("content-length:"): + length = int(line.split(":", 1)[1].strip()) + break + assert length is not None + body_start = header_end + 4 + body_end = body_start + length + messages.append(json.loads(data[body_start:body_end].decode("utf-8"))) + cursor = body_end + return messages + + def _http_rpc( host: str, port: int, diff --git a/tests/test_search.py b/tests/test_search.py index 859cbe0..3635770 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -11,7 +11,7 @@ from codebase_graph.db import GraphNeighbor, SearchIndexRow from codebase_graph.ingest import GraphMaterializer from codebase_graph.mcp.runtime import GraphRuntimeConfig -from codebase_graph.mcp.tools import handle_tool_call +from codebase_graph.mcp.tools import MAX_GRAPH_QUERY_LIMIT, _query_payload, handle_tool_call from codebase_graph.reasoning import CompactContextBuilder, ContextNode from codebase_graph.retrieval.search import CompactContextPayload, SearchHit, SearchRequest, SearchService @@ -19,10 +19,19 @@ class _Result: def __init__(self, rows: list[list[Any]]) -> None: self.rows = rows + self.requested_n: int | None = None + self.closed = False def get_all(self) -> list[list[Any]]: return self.rows + def get_n(self, count: int) -> list[list[Any]]: + self.requested_n = count + return self.rows[:count] + + def close(self) -> None: + self.closed = True + class _RecordingStore: def __init__(self, rows: list[list[Any]] | None = None) -> None: @@ -31,7 +40,8 @@ def __init__(self, rows: list[list[Any]] | None = None) -> None: def execute(self, statement: str, parameters: dict[str, Any] | None = None) -> _Result: self.calls.append((statement, parameters)) - return _Result(self.rows) + self.result = _Result(self.rows) + return self.result class _Adapter: @@ -594,6 +604,30 @@ def test_cli_graph_query_rejects_write_like_statements(tmp_path: Path) -> None: assert exc_info.value.code == 2 +def test_graph_query_fetches_limit_plus_one_rows_without_materializing_all() -> None: + store = _RecordingStore([[1], [2], [3], [4]]) + + payload = _query_payload(store, {"statement": "MATCH (n) RETURN n", "limit": 2}) + + assert store.result.requested_n == 3 + assert store.result.closed is True + assert payload == { + "statement": "MATCH (n) RETURN n", + "row_count": 2, + "rows": [[1], [2]], + "truncated": True, + } + + +def test_graph_query_rejects_unbounded_response_limits() -> None: + store = _RecordingStore([[1]]) + + with pytest.raises(ValueError, match="greater than zero"): + _query_payload(store, {"statement": "MATCH (n) RETURN n", "limit": 0}) + with pytest.raises(ValueError, match=f"{MAX_GRAPH_QUERY_LIMIT} or less"): + _query_payload(store, {"statement": "MATCH (n) RETURN n", "limit": MAX_GRAPH_QUERY_LIMIT + 1}) + + def _require_graph_runtime() -> None: pytest.importorskip("tree_sitter") pytest.importorskip("tree_sitter_python") diff --git a/tests/test_setup_workflow.py b/tests/test_setup_workflow.py index bfc6f9e..16a9499 100644 --- a/tests/test_setup_workflow.py +++ b/tests/test_setup_workflow.py @@ -92,12 +92,16 @@ def test_setup_cli_creates_state_db_mcp_config_instructions_and_searchable_docs( {"query": "codebaseGraph workflow", "profile": "docs", "limit": 5}, runtime=server.runtime, ) + health_payload = handle_tool_call("graph_health", {}, runtime=server.runtime) symbol_payload = handle_tool_call( "graph_search", {"query": "SampleService", "profile": "brief", "limit": 3}, runtime=server.runtime, ) + assert health_payload["ok"] is True + assert health_payload["graph_readable"] is True + assert health_payload["total_nodes"] > 0 assert any(hit["path"] == "AGENTS.md" for hit in docs_payload["results"]) assert any(hit["label"] == "SampleService" for hit in symbol_payload["results"]) From eb12ff5d5a3d6f2855f8762b474b947e72c02851 Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Wed, 27 May 2026 13:57:45 +0930 Subject: [PATCH 26/53] chore: raise production readiness evidence The runtime now emits structured operational diagnostics for setup, MCP transport failures, and materialization lock conflicts. CI and release workflows also validate dependency integrity, build an SBOM, and attach that artifact to releases so package consumers have a clearer audit trail. Constraint: Real vulnerability advisory queries send dependency inventory externally, so local verification uses dry-run dependency collection and SBOM generation. Rejected: Run pip-audit advisory queries implicitly during local setup | external dependency inventory disclosure requires explicit authorization. Confidence: high Scope-risk: moderate Directive: Keep external advisory scans explicit; do not make local setup call advisory APIs silently. Tested: ./.venv/bin/python -m pytest Tested: ./.venv/bin/ruff check . Tested: ./.venv/bin/python -m build --no-isolation --outdir /private/tmp/codebasegraph-dist-review-2 Tested: ./.venv/bin/python -m twine check /private/tmp/codebasegraph-dist-review-2/* Tested: ./.venv/bin/python scripts/smoke_built_wheel.py ./.venv/bin/codebase-graph Tested: pip check, pip-audit dry-run, CycloneDX SBOM generation, workflow YAML parse, diagnostics smoke, graph refresh. Not-tested: Hosted GitHub CI execution and external vulnerability advisory scan. --- .github/workflows/ci.yml | 44 ++++++++++++++++ .github/workflows/release.yml | 18 ++++++- README.md | 15 +++++- docs/release.md | 10 +++- src/codebase_graph/diagnostics.py | 58 ++++++++++++++++++++++ src/codebase_graph/ingest/materializer.py | 7 +++ src/codebase_graph/mcp/tools.py | 15 ++++++ src/codebase_graph/mcp/transports/http.py | 33 ++++++++++++ src/codebase_graph/mcp/transports/stdio.py | 2 + src/codebase_graph/setup/orchestrator.py | 24 +++++++++ tests/test_diagnostics.py | 31 ++++++++++++ tests/test_mcp_portability.py | 3 +- 12 files changed, 255 insertions(+), 5 deletions(-) create mode 100644 src/codebase_graph/diagnostics.py create mode 100644 tests/test_diagnostics.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f78eafa..262093d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,6 +75,50 @@ jobs: - name: Run ruff run: ruff check . + supply-chain: + name: supply chain + runs-on: ubuntu-latest + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + + - name: Install package and supply-chain tools + run: | + python -m pip install --upgrade pip + python -m pip install -e ".[dev]" + python -m pip install cyclonedx-bom pip-audit + + - name: Check installed dependency consistency + run: python -m pip check + + - name: Validate audit dependency collection + run: python -m pip_audit --strict --dry-run --skip-editable --progress-spinner off --cache-dir .pip-audit-cache + + - name: Generate CycloneDX SBOM + run: | + cyclonedx-py environment "$(which python)" \ + --pyproject pyproject.toml \ + --mc-type library \ + --output-reproducible \ + --of JSON \ + -o codebase-graph-sbom.cdx.json + + - name: Upload SBOM + uses: actions/upload-artifact@v4 + with: + name: codebase-graph-sbom + path: codebase-graph-sbom.cdx.json + if-no-files-found: error + package: name: package runs-on: ubuntu-latest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 871d09d..10d7a2e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -108,9 +108,18 @@ jobs: python -m venv /tmp/codebase-graph-wheel /tmp/codebase-graph-wheel/bin/python -m pip install --upgrade pip /tmp/codebase-graph-wheel/bin/python -m pip install dist/*.whl + /tmp/codebase-graph-wheel/bin/python -m pip install cyclonedx-bom pip-audit /tmp/codebase-graph-wheel/bin/codebase-graph --help /tmp/codebase-graph-wheel/bin/codebase-graph-mcp --help /tmp/codebase-graph-wheel/bin/python scripts/smoke_built_wheel.py /tmp/codebase-graph-wheel/bin/codebase-graph + /tmp/codebase-graph-wheel/bin/python -m pip check + /tmp/codebase-graph-wheel/bin/python -m pip_audit --strict --dry-run --skip-editable --progress-spinner off --cache-dir /tmp/pip-audit-cache + /tmp/codebase-graph-wheel/bin/cyclonedx-py environment /tmp/codebase-graph-wheel/bin/python \ + --pyproject pyproject.toml \ + --mc-type library \ + --output-reproducible \ + --of JSON \ + -o codebase-graph-${{ steps.verify-version.outputs.package-version }}-sbom.cdx.json - name: Upload distributions uses: actions/upload-artifact@v4 @@ -119,10 +128,17 @@ jobs: path: dist/* if-no-files-found: error + - name: Upload SBOM artifact + uses: actions/upload-artifact@v4 + with: + name: codebase-graph-${{ steps.verify-version.outputs.package-version }}-sbom + path: codebase-graph-${{ steps.verify-version.outputs.package-version }}-sbom.cdx.json + if-no-files-found: error + - name: Upload distributions to GitHub release env: GH_TOKEN: ${{ github.token }} - run: gh release upload "$RELEASE_TAG" dist/* --clobber + run: gh release upload "$RELEASE_TAG" dist/* codebase-graph-${{ steps.verify-version.outputs.package-version }}-sbom.cdx.json --clobber publish-pypi: name: publish to PyPI diff --git a/README.md b/README.md index 5c7db19..9d0598e 100644 --- a/README.md +++ b/README.md @@ -162,6 +162,19 @@ truncated. Add a narrower `MATCH` pattern or a query-side `LIMIT` for broader gr For coding-task architecture orientation, call `graph_architecture_queries` first to fetch the grouped read-only Cypher catalog, then run selected statements with `graph_query`. +## Operational diagnostics + +Runtime warning and error paths emit structured JSON events to stderr. Set `CODEBASE_GRAPH_LOG_LEVEL=INFO` to include +setup start/completion diagnostics; the default level is `WARNING`. + +Examples of emitted events include: + +- `setup.failed` +- `mcp.tool_error` +- `mcp.stdio_parse_error` +- `mcp.http_forbidden_origin` +- `materializer.lock_exists` + ## CLI graph workflow The CLI exposes the same graph workflow as the MCP tools, which is useful in clients that do not surface MCP tools directly: @@ -202,7 +215,7 @@ ruff check . ## CI and releases -GitHub Actions runs pytest across Linux, macOS, and Windows for Python 3.10 through 3.14, plus ruff and package-build validation. Built wheels are smoke-tested with `setup`, `graph-health`, `graph-search`, and a stdio MCP handshake before release. Releases are managed by release-please, use tag-derived package versions, create GitHub Releases with distribution assets, and publish to PyPI through Trusted Publishing. +GitHub Actions runs pytest across Linux, macOS, and Windows for Python 3.10 through 3.14, plus ruff, supply-chain, and package-build validation. Supply-chain checks include dependency consistency, audit dependency collection, Dependabot update coverage, and CycloneDX SBOM generation. Built wheels are smoke-tested with `setup`, `graph-health`, `graph-search`, and a stdio MCP handshake before release. Releases are managed by release-please, use tag-derived package versions, create GitHub Releases with distribution assets and SBOMs, and publish to PyPI through Trusted Publishing. Conda distribution uses the conda-forge staged-recipes path rather than direct Anaconda.org uploads. See [docs/release.md](docs/release.md) for the release workflow and conda-forge submission checklist. diff --git a/docs/release.md b/docs/release.md index b9b7727..f7d0fa0 100644 --- a/docs/release.md +++ b/docs/release.md @@ -19,15 +19,21 @@ Pull requests and pushes to `main` or `codex/**` run: - `pytest` on Linux, macOS, and Windows for Python 3.10 through 3.14. - `ruff check .` on Linux. +- Supply-chain checks on Linux with `pip check`, `pip-audit --dry-run --strict` dependency collection, and CycloneDX + SBOM generation. - A package build on Linux with `python -m build`, `twine check`, console-script smoke tests from the built wheel, - and a packaged runtime smoke that runs `setup`, `graph-health`, `graph-search`, and stdio MCP handshake checks. + packaged runtime smoke that runs `setup`, `graph-health`, `graph-search`, and stdio MCP handshake checks, and release + SBOM generation. ## Release flow 1. Merge normal pull requests into `main` with Conventional Commit-style titles or squash commit messages such as `feat: add graph query helpers` or `fix: preserve MCP config`. 2. The `Release` workflow opens or updates a release pull request that updates `CHANGELOG.md` and `.release-please-manifest.json`. 3. Review and merge the release pull request when ready to publish. -4. The `Release` workflow creates the `vX.Y.Z` tag and GitHub Release, builds the distributions from that tag, verifies `Version: X.Y.Z`, uploads the distributions to the GitHub Release, and publishes to PyPI from the protected `pypi` environment. +4. The `Release` workflow creates the `vX.Y.Z` tag and GitHub Release, builds the distributions from that tag, verifies `Version: X.Y.Z`, uploads the distributions and SBOM to the GitHub Release, and publishes to PyPI from the protected `pypi` environment. + +Vulnerability advisory scans require an external advisory service. Keep them in the hosted CI/release environment or +run them explicitly from a development machine; do not make local setup call external advisory APIs implicitly. The package version remains tag-derived through `setuptools_scm`; do not add a static `project.version` field to `pyproject.toml` just for release-please. diff --git a/src/codebase_graph/diagnostics.py b/src/codebase_graph/diagnostics.py new file mode 100644 index 0000000..ec4e5ca --- /dev/null +++ b/src/codebase_graph/diagnostics.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +import json +import os +import sys +from datetime import datetime, timezone +from typing import Any + +LOG_LEVEL_ENV = "CODEBASE_GRAPH_LOG_LEVEL" +_LEVELS = { + "DEBUG": 10, + "INFO": 20, + "WARNING": 30, + "ERROR": 40, + "CRITICAL": 50, +} + + +def log_event(event: str, *, level: str = "INFO", **fields: Any) -> None: + normalized_level = level.upper() + if _LEVELS.get(normalized_level, 20) < _configured_level(): + return + payload = { + "event": event, + "level": normalized_level, + "timestamp": datetime.now(timezone.utc).isoformat(), + **_safe_fields(fields), + } + print(json.dumps(payload, separators=(",", ":"), sort_keys=True), file=sys.stderr) + + +def _configured_level() -> int: + configured = os.environ.get(LOG_LEVEL_ENV, "WARNING").upper() + return _LEVELS.get(configured, _LEVELS["WARNING"]) + + +def _safe_fields(fields: dict[str, Any]) -> dict[str, Any]: + safe: dict[str, Any] = {} + for key, value in fields.items(): + if value is None or isinstance(value, (str, int, float, bool)): + safe[key] = value + elif isinstance(value, (list, tuple)): + safe[key] = [_safe_value(item) for item in value] + elif isinstance(value, dict): + safe[key] = {str(item_key): _safe_value(item_value) for item_key, item_value in value.items()} + else: + safe[key] = str(value) + return safe + + +def _safe_value(value: Any) -> Any: + if value is None or isinstance(value, (str, int, float, bool)): + return value + if isinstance(value, (list, tuple)): + return [_safe_value(item) for item in value] + if isinstance(value, dict): + return {str(key): _safe_value(item) for key, item in value.items()} + return str(value) diff --git a/src/codebase_graph/ingest/materializer.py b/src/codebase_graph/ingest/materializer.py index 30f8075..d7f52f8 100644 --- a/src/codebase_graph/ingest/materializer.py +++ b/src/codebase_graph/ingest/materializer.py @@ -12,6 +12,7 @@ from codebase_graph.core import CodeGraph from codebase_graph.db import LadybugCodeGraphStore, create_ladybug_database +from codebase_graph.diagnostics import log_event from codebase_graph.extract import GraphBuilder from codebase_graph.ontology import ONTOLOGY_NAME from codebase_graph.paths import DEFAULT_STATE_DIR, derive_graph_state_paths @@ -509,6 +510,12 @@ def _acquire_materialization_lock(db_path: Path) -> tuple[int, Path]: try: descriptor = os.open(lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY) except FileExistsError as exc: + log_event( + "materializer.lock_exists", + level="WARNING", + db_path=db_path.as_posix(), + lock_path=lock_path.as_posix(), + ) raise RuntimeError( f"codebaseGraph materialization is already in progress for {db_path}. " f"If no materializer is running, remove the stale lock file: {lock_path}" diff --git a/src/codebase_graph/mcp/tools.py b/src/codebase_graph/mcp/tools.py index 76f9df4..374bb36 100644 --- a/src/codebase_graph/mcp/tools.py +++ b/src/codebase_graph/mcp/tools.py @@ -5,6 +5,7 @@ from typing import Any from codebase_graph.db import LadybugCodeGraphStore +from codebase_graph.diagnostics import log_event from codebase_graph.ontology import QUERY_HELPERS, schema_payload from codebase_graph.reasoning import CompactContextBuilder, architecture_query_catalog from codebase_graph.retrieval import DETAIL_LEVELS, SearchRequest, SearchService @@ -63,6 +64,13 @@ def tool_result(payload: dict[str, Any]) -> dict[str, Any]: def tool_error_result(name: str, exc: Exception) -> dict[str, Any]: + log_event( + "mcp.tool_error", + level="WARNING", + tool=name, + error_type=exc.__class__.__name__, + message=str(exc), + ) payload = { "error": { "tool": name, @@ -152,6 +160,13 @@ def _health(runtime: GraphRuntimeConfig) -> dict[str, Any]: except Exception as exc: payload["graph_readable"] = False payload["error"] = {"type": exc.__class__.__name__, "message": str(exc)} + log_event( + "mcp.graph_health_failed", + level="WARNING", + database_path=runtime.db_path.as_posix(), + error_type=exc.__class__.__name__, + message=str(exc), + ) return payload payload["ok"] = True payload["graph_readable"] = True diff --git a/src/codebase_graph/mcp/transports/http.py b/src/codebase_graph/mcp/transports/http.py index 200f628..b6c3903 100644 --- a/src/codebase_graph/mcp/transports/http.py +++ b/src/codebase_graph/mcp/transports/http.py @@ -7,6 +7,7 @@ from typing import Any from urllib.parse import urlparse +from codebase_graph.diagnostics import log_event from codebase_graph.mcp.protocol import SUPPORTED_PROTOCOL_VERSIONS, McpGraphServer, rpc_error LOCAL_ORIGINS = {"localhost", "127.0.0.1", "::1"} @@ -32,6 +33,7 @@ def build_http_server( allow_remote: bool = False, ) -> McpHttpServer: if not allow_remote and host not in LOCAL_ORIGINS: + log_event("mcp.http_remote_bind_rejected", level="WARNING", host=host, port=port) raise ValueError("MCP HTTP transport may only bind to localhost unless allow_remote is enabled") graph_server = McpGraphServer.from_paths( repo_root=repo_root, @@ -86,6 +88,7 @@ def do_POST(self) -> None: try: message = json.loads(self.rfile.read(length).decode("utf-8")) except Exception as exc: + log_event("mcp.http_parse_error", level="WARNING", message=str(exc), client_address=self.client_address[0]) self._send_json(rpc_error(None, -32700, f"Invalid JSON-RPC payload: {exc}"), status=HTTPStatus.BAD_REQUEST) return if not isinstance(message, dict): @@ -121,6 +124,12 @@ def _valid_origin(self) -> bool: hostname = urlparse(origin).hostname if hostname in LOCAL_ORIGINS: return True + log_event( + "mcp.http_forbidden_origin", + level="WARNING", + origin=origin, + client_address=self.client_address[0], + ) self._send_json(rpc_error(None, -32000, "Forbidden origin"), status=HTTPStatus.FORBIDDEN) return False @@ -130,6 +139,12 @@ def _valid_protocol_header(self) -> bool: return True if requested in SUPPORTED_PROTOCOL_VERSIONS: return True + log_event( + "mcp.http_unsupported_protocol", + level="WARNING", + requested=requested, + client_address=self.client_address[0], + ) self._send_json( rpc_error( None, @@ -146,12 +161,30 @@ def _content_length(self) -> int | None: try: length = int(raw_length) except ValueError: + log_event( + "mcp.http_invalid_content_length", + level="WARNING", + content_length=raw_length, + client_address=self.client_address[0], + ) self._send_json(rpc_error(None, -32600, "Content-Length must be an integer"), status=HTTPStatus.BAD_REQUEST) return None if length < 0: + log_event( + "mcp.http_invalid_content_length", + level="WARNING", + content_length=raw_length, + client_address=self.client_address[0], + ) self._send_json(rpc_error(None, -32600, "Content-Length must be non-negative"), status=HTTPStatus.BAD_REQUEST) return None if length > MAX_HTTP_BODY_BYTES: + log_event( + "mcp.http_body_too_large", + level="WARNING", + content_length=length, + client_address=self.client_address[0], + ) self._send_json( rpc_error(None, -32000, "MCP request body is too large", {"max_bytes": MAX_HTTP_BODY_BYTES}), status=HTTPStatus.REQUEST_ENTITY_TOO_LARGE, diff --git a/src/codebase_graph/mcp/transports/stdio.py b/src/codebase_graph/mcp/transports/stdio.py index 8a063d3..8736841 100644 --- a/src/codebase_graph/mcp/transports/stdio.py +++ b/src/codebase_graph/mcp/transports/stdio.py @@ -5,6 +5,7 @@ from pathlib import Path from typing import Any, BinaryIO +from codebase_graph.diagnostics import log_event from codebase_graph.mcp.protocol import McpGraphServer, rpc_error @@ -29,6 +30,7 @@ def serve_stdio( try: message = read_message(sys.stdin.buffer) except StdioMessageError as exc: + log_event("mcp.stdio_parse_error", level="WARNING", message=str(exc)) write_message(sys.stdout.buffer, rpc_error(None, -32700, f"Invalid JSON-RPC payload: {exc}")) continue if message is None: diff --git a/src/codebase_graph/setup/orchestrator.py b/src/codebase_graph/setup/orchestrator.py index 7bf8b8b..bc9deb4 100644 --- a/src/codebase_graph/setup/orchestrator.py +++ b/src/codebase_graph/setup/orchestrator.py @@ -4,6 +4,7 @@ from pathlib import Path from typing import Any +from codebase_graph.diagnostics import log_event from codebase_graph.ingest import GraphMaterializer from .instructions import InstructionResult, upsert_instruction_block @@ -49,6 +50,13 @@ def as_dict(self) -> dict[str, Any]: def run_setup(options: SetupOptions) -> SetupResult: try: + log_event( + "setup.start", + level="INFO", + repo_root=str(options.repo_root), + mcp_client=options.mcp_client, + dry_run=options.dry_run, + ) paths = derive_setup_paths(options.repo_root) validate_ladybug_runtime() paths.state_dir.mkdir(parents=True, exist_ok=True) @@ -81,9 +89,25 @@ def run_setup(options: SetupOptions) -> SetupResult: skip=options.skip_mcp_config, ) except Exception as exc: + log_event( + "setup.failed", + level="ERROR", + repo_root=str(options.repo_root), + error_type=exc.__class__.__name__, + message=str(exc), + ) if isinstance(exc, SetupError): raise raise SetupError(str(exc)) from exc + log_event( + "setup.completed", + level="INFO", + repo_root=paths.repo_root.as_posix(), + config_action=config_action, + rebuilt=getattr(materialization, "rebuilt"), + deleted=getattr(materialization, "deleted"), + mcp_action=mcp_result.action, + ) return SetupResult( paths=paths, config_action=config_action, diff --git a/tests/test_diagnostics.py b/tests/test_diagnostics.py new file mode 100644 index 0000000..7673762 --- /dev/null +++ b/tests/test_diagnostics.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import json + +from codebase_graph.diagnostics import LOG_LEVEL_ENV, log_event + + +def test_log_event_emits_json_to_stderr_when_level_allows( + monkeypatch, + capsys, +) -> None: + monkeypatch.setenv(LOG_LEVEL_ENV, "INFO") + + log_event("sample.event", level="INFO", count=2, payload={"ok": True}) + + captured = capsys.readouterr() + assert captured.out == "" + event = json.loads(captured.err) + assert event["event"] == "sample.event" + assert event["level"] == "INFO" + assert event["count"] == 2 + assert event["payload"] == {"ok": True} + assert event["timestamp"] + + +def test_log_event_respects_configured_level(monkeypatch, capsys) -> None: + monkeypatch.setenv(LOG_LEVEL_ENV, "ERROR") + + log_event("sample.event", level="INFO") + + assert capsys.readouterr().err == "" diff --git a/tests/test_mcp_portability.py b/tests/test_mcp_portability.py index 2639584..50cb84e 100644 --- a/tests/test_mcp_portability.py +++ b/tests/test_mcp_portability.py @@ -210,7 +210,8 @@ def test_stdio_mcp_malformed_frame_returns_parse_error(tmp_path: Path) -> None: responses = _stdio_messages(completed.stdout) assert completed.returncode == 0 - assert completed.stderr == b"" + stderr_events = [json.loads(line) for line in completed.stderr.decode("utf-8").splitlines()] + assert stderr_events[0]["event"] == "mcp.stdio_parse_error" assert responses[0]["error"]["code"] == -32700 From 0893032b747b27b7b80a2154cfafcf9e0c22f509 Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Wed, 27 May 2026 16:11:01 +0930 Subject: [PATCH 27/53] fix: close production readiness blockers Setup now has stricter repository-root invariants, honest dry-run behavior, and rollback for published control files when materialization fails. The graph query surface blocks procedure calls, HTTP no longer shares one mutable MCP session across handlers, materialization skips more generated directories and avoids hashing unsupported files, and release workflows now use immutable Action pins, real hosted advisory scans, and wheel plus sdist smoke coverage. Constraint: SPDX license selection and PyPI/GitHub environment setup are owner-controlled release decisions and cannot be invented in code. Constraint: Local advisory scans disclose dependency inventory externally, so local verification used pip-audit dry-run while hosted workflows run real scans. Rejected: Keep pip-audit dry-run in hosted CI | it cannot fail releases on known vulnerable dependencies. Rejected: Preserve setup --dry-run side effects | dry-run should not publish repository or client state. Rejected: Hash unsupported files before skipping them | large binary/vendor trees make that unsuitable for production repositories. Confidence: high Scope-risk: moderate Directive: Do not relax graph_query CALL blocking without a parser-level read-only proof or an explicit safe-procedure allowlist. Directive: Keep GitHub Actions pinned to immutable commits and let review policy decide update cadence. Tested: ./.venv/bin/python -m pytest Tested: ./.venv/bin/ruff check . Tested: ruby YAML parse for CI, release, and Dependabot workflows Tested: ./.venv/bin/python -m build --no-isolation --outdir /private/tmp/codebasegraph-dist-prod-ready-4 Tested: ./.venv/bin/python -m twine check /private/tmp/codebasegraph-dist-prod-ready-4/* Tested: wheel and source-distribution smoke tests using scripts/smoke_built_wheel.py against built artifacts Tested: ./.venv/bin/python -m pip check Tested: pip-audit dry-run, CycloneDX SBOM generation, codebase-graph setup refresh, graph-query count Not-tested: Hosted GitHub CI execution, real external vulnerability advisory query from local machine, PyPI Trusted Publisher setup, project license selection. --- .github/workflows/ci.yml | 32 +++-- .github/workflows/release.yml | 26 ++-- README.md | 4 +- docs/release.md | 24 +++- src/codebase_graph/ingest/materializer.py | 26 +++- src/codebase_graph/mcp/tools.py | 2 +- src/codebase_graph/mcp/transports/http.py | 9 +- src/codebase_graph/setup/instructions.py | 6 + src/codebase_graph/setup/orchestrator.py | 162 ++++++++++++++++++---- src/codebase_graph/setup/state.py | 4 + tests/test_materializer.py | 31 ++++- tests/test_release_workflows.py | 39 ++++++ tests/test_search.py | 7 + tests/test_setup_workflow.py | 61 ++++++++ 14 files changed, 367 insertions(+), 66 deletions(-) create mode 100644 tests/test_release_workflows.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 262093d..4d5b7ae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,12 +33,12 @@ jobs: steps: - name: Check out repository - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: ${{ matrix.python-version }} cache: pip @@ -57,12 +57,12 @@ jobs: steps: - name: Check out repository - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.12" cache: pip @@ -81,12 +81,12 @@ jobs: steps: - name: Check out repository - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.12" cache: pip @@ -100,8 +100,8 @@ jobs: - name: Check installed dependency consistency run: python -m pip check - - name: Validate audit dependency collection - run: python -m pip_audit --strict --dry-run --skip-editable --progress-spinner off --cache-dir .pip-audit-cache + - name: Run vulnerability advisory scan + run: python -m pip_audit --strict --skip-editable --progress-spinner off --cache-dir .pip-audit-cache - name: Generate CycloneDX SBOM run: | @@ -113,7 +113,7 @@ jobs: -o codebase-graph-sbom.cdx.json - name: Upload SBOM - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: codebase-graph-sbom path: codebase-graph-sbom.cdx.json @@ -125,12 +125,12 @@ jobs: steps: - name: Check out repository - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.12" cache: pip @@ -151,3 +151,13 @@ jobs: /tmp/codebase-graph-wheel/bin/codebase-graph --help /tmp/codebase-graph-wheel/bin/codebase-graph-mcp --help /tmp/codebase-graph-wheel/bin/python scripts/smoke_built_wheel.py /tmp/codebase-graph-wheel/bin/codebase-graph + + - name: Smoke-test source distribution + shell: bash + run: | + python -m venv /tmp/codebase-graph-sdist + /tmp/codebase-graph-sdist/bin/python -m pip install --upgrade pip + /tmp/codebase-graph-sdist/bin/python -m pip install dist/*.tar.gz + /tmp/codebase-graph-sdist/bin/codebase-graph --help + /tmp/codebase-graph-sdist/bin/codebase-graph-mcp --help + /tmp/codebase-graph-sdist/bin/python scripts/smoke_built_wheel.py /tmp/codebase-graph-sdist/bin/codebase-graph diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 10d7a2e..8a1234a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,7 +24,7 @@ jobs: steps: - name: Create release pull request or GitHub release id: release - uses: googleapis/release-please-action@v4 + uses: googleapis/release-please-action@5c625bfb5d1ff62eadeeb3772007f7f66fdcf071 # v4 with: token: ${{ secrets.RELEASE_PLEASE_TOKEN || github.token }} config-file: release-please-config.json @@ -45,13 +45,13 @@ jobs: steps: - name: Check out release tag - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: ref: ${{ env.RELEASE_TAG }} fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.12" cache: pip @@ -113,7 +113,7 @@ jobs: /tmp/codebase-graph-wheel/bin/codebase-graph-mcp --help /tmp/codebase-graph-wheel/bin/python scripts/smoke_built_wheel.py /tmp/codebase-graph-wheel/bin/codebase-graph /tmp/codebase-graph-wheel/bin/python -m pip check - /tmp/codebase-graph-wheel/bin/python -m pip_audit --strict --dry-run --skip-editable --progress-spinner off --cache-dir /tmp/pip-audit-cache + /tmp/codebase-graph-wheel/bin/python -m pip_audit --strict --skip-editable --progress-spinner off --cache-dir /tmp/pip-audit-cache /tmp/codebase-graph-wheel/bin/cyclonedx-py environment /tmp/codebase-graph-wheel/bin/python \ --pyproject pyproject.toml \ --mc-type library \ @@ -121,15 +121,25 @@ jobs: --of JSON \ -o codebase-graph-${{ steps.verify-version.outputs.package-version }}-sbom.cdx.json + - name: Smoke-test source distribution + shell: bash + run: | + python -m venv /tmp/codebase-graph-sdist + /tmp/codebase-graph-sdist/bin/python -m pip install --upgrade pip + /tmp/codebase-graph-sdist/bin/python -m pip install dist/*.tar.gz + /tmp/codebase-graph-sdist/bin/codebase-graph --help + /tmp/codebase-graph-sdist/bin/codebase-graph-mcp --help + /tmp/codebase-graph-sdist/bin/python scripts/smoke_built_wheel.py /tmp/codebase-graph-sdist/bin/codebase-graph + - name: Upload distributions - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: codebase-graph-${{ steps.verify-version.outputs.package-version }}-dist path: dist/* if-no-files-found: error - name: Upload SBOM artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: codebase-graph-${{ steps.verify-version.outputs.package-version }}-sbom path: codebase-graph-${{ steps.verify-version.outputs.package-version }}-sbom.cdx.json @@ -155,10 +165,10 @@ jobs: steps: - name: Download distributions - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: name: codebase-graph-${{ needs.build.outputs.package-version }}-dist path: dist - name: Publish distributions to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1 diff --git a/README.md b/README.md index 9d0598e..ac88199 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ codebase-graph setup --skip-mcp-config codebase-graph setup --instructions-target claude ``` -`--dry-run` returns the raw server descriptor plus the exact client patch or payload without writing the MCP client file. Repository graph state and instruction handling still run so the graph can be verified. +`--dry-run` returns the raw server descriptor plus the exact client patch or payload without writing repository graph state, instruction files, or MCP client files. ## MCP installation @@ -215,7 +215,7 @@ ruff check . ## CI and releases -GitHub Actions runs pytest across Linux, macOS, and Windows for Python 3.10 through 3.14, plus ruff, supply-chain, and package-build validation. Supply-chain checks include dependency consistency, audit dependency collection, Dependabot update coverage, and CycloneDX SBOM generation. Built wheels are smoke-tested with `setup`, `graph-health`, `graph-search`, and a stdio MCP handshake before release. Releases are managed by release-please, use tag-derived package versions, create GitHub Releases with distribution assets and SBOMs, and publish to PyPI through Trusted Publishing. +GitHub Actions runs pytest across Linux, macOS, and Windows for Python 3.10 through 3.14, plus ruff, supply-chain, and package-build validation. Supply-chain checks include dependency consistency, vulnerability advisory scanning, Dependabot update coverage, immutable GitHub Action pins, and CycloneDX SBOM generation. Built wheels and source distributions are smoke-tested with `setup`, `graph-health`, `graph-search`, and a stdio MCP handshake before release. Releases are managed by release-please, use tag-derived package versions, create GitHub Releases with distribution assets and SBOMs, and publish to PyPI through Trusted Publishing. Conda distribution uses the conda-forge staged-recipes path rather than direct Anaconda.org uploads. See [docs/release.md](docs/release.md) for the release workflow and conda-forge submission checklist. diff --git a/docs/release.md b/docs/release.md index f7d0fa0..40c8cda 100644 --- a/docs/release.md +++ b/docs/release.md @@ -19,11 +19,11 @@ Pull requests and pushes to `main` or `codex/**` run: - `pytest` on Linux, macOS, and Windows for Python 3.10 through 3.14. - `ruff check .` on Linux. -- Supply-chain checks on Linux with `pip check`, `pip-audit --dry-run --strict` dependency collection, and CycloneDX - SBOM generation. -- A package build on Linux with `python -m build`, `twine check`, console-script smoke tests from the built wheel, - packaged runtime smoke that runs `setup`, `graph-health`, `graph-search`, and stdio MCP handshake checks, and release - SBOM generation. +- Supply-chain checks on Linux with `pip check`, `pip-audit --strict` vulnerability advisory scanning, immutable + GitHub Action pins, and CycloneDX SBOM generation. +- A package build on Linux with `python -m build`, `twine check`, console-script smoke tests from the built wheel and + source distribution, packaged runtime smoke that runs `setup`, `graph-health`, `graph-search`, and stdio MCP handshake + checks, and release SBOM generation. ## Release flow @@ -32,8 +32,18 @@ Pull requests and pushes to `main` or `codex/**` run: 3. Review and merge the release pull request when ready to publish. 4. The `Release` workflow creates the `vX.Y.Z` tag and GitHub Release, builds the distributions from that tag, verifies `Version: X.Y.Z`, uploads the distributions and SBOM to the GitHub Release, and publishes to PyPI from the protected `pypi` environment. -Vulnerability advisory scans require an external advisory service. Keep them in the hosted CI/release environment or -run them explicitly from a development machine; do not make local setup call external advisory APIs implicitly. +## Release gate + +Before publishing a production release, confirm: + +- Hosted CI is green for tests, ruff, package build, supply-chain, wheel smoke, and source-distribution smoke. +- The project owner has selected an SPDX license, added package license metadata, and included the corresponding license file. +- The PyPI Trusted Publisher, `pypi` GitHub environment, and release-please token posture have been verified in GitHub/PyPI settings. +- Conda-forge submission is either explicitly out of scope for the release or the recipe placeholders have been replaced with the release version, source-distribution SHA256, and chosen SPDX license. + +Vulnerability advisory scans require an external advisory service. Hosted CI and release workflows run those scans and +fail on known vulnerable dependencies. Local setup stays offline-safe and must not call external advisory APIs +implicitly; run local advisory scans explicitly when that disclosure is acceptable. The package version remains tag-derived through `setuptools_scm`; do not add a static `project.version` field to `pyproject.toml` just for release-please. diff --git a/src/codebase_graph/ingest/materializer.py b/src/codebase_graph/ingest/materializer.py index d7f52f8..9c61757 100644 --- a/src/codebase_graph/ingest/materializer.py +++ b/src/codebase_graph/ingest/materializer.py @@ -26,11 +26,24 @@ EXCLUDED_PARTS = { ".git", ".venv", - "__pycache__", + ".cache", + ".coverage", + ".hypothesis", + ".mypy_cache", + ".nox", + ".pyre", ".pytest_cache", + ".pytype", ".ruff_cache", + ".tox", + ".vscode", + "__pycache__", "build", + "coverage", "dist", + "htmlcov", + "node_modules", + "vendor", ".codebase_graph", DEFAULT_STATE_DIR, } @@ -448,14 +461,21 @@ def _scan_source_files(self) -> tuple[dict[str, SourceSnapshot], list[str]]: continue relative_path = path.relative_to(self.source_root).as_posix() language = self.parser_registry.language_for_path(path) + if language is None: + snapshots[relative_path] = SourceSnapshot( + path=relative_path, + absolute_path=path, + content_hash="", + language=None, + ) + diagnostics.append(f"Skipped unsupported file: {relative_path}") + continue snapshots[relative_path] = SourceSnapshot( path=relative_path, absolute_path=path, content_hash=_file_hash(path), language=language, ) - if language is None: - diagnostics.append(f"Skipped unsupported file: {relative_path}") return snapshots, diagnostics def _build_graph(self, snapshot: SourceSnapshot) -> CodeGraph: diff --git a/src/codebase_graph/mcp/tools.py b/src/codebase_graph/mcp/tools.py index 374bb36..ba74fbe 100644 --- a/src/codebase_graph/mcp/tools.py +++ b/src/codebase_graph/mcp/tools.py @@ -13,7 +13,7 @@ from .runtime import GraphRuntimeConfig, open_graph_store READ_ONLY_DENY_RE = re.compile( - r"\b(CREATE|DELETE|SET|MERGE|DROP|COPY|INSERT|LOAD|INSTALL|DETACH|REMOVE|ALTER|RENAME)\b", + r"\b(CALL|CREATE|DELETE|SET|MERGE|DROP|COPY|INSERT|LOAD|INSTALL|DETACH|REMOVE|ALTER|RENAME)\b", re.IGNORECASE, ) MAX_GRAPH_QUERY_LIMIT = 1000 diff --git a/src/codebase_graph/mcp/transports/http.py b/src/codebase_graph/mcp/transports/http.py index b6c3903..50924f0 100644 --- a/src/codebase_graph/mcp/transports/http.py +++ b/src/codebase_graph/mcp/transports/http.py @@ -9,6 +9,7 @@ from codebase_graph.diagnostics import log_event from codebase_graph.mcp.protocol import SUPPORTED_PROTOCOL_VERSIONS, McpGraphServer, rpc_error +from codebase_graph.mcp.runtime import GraphRuntimeConfig, runtime_config LOCAL_ORIGINS = {"localhost", "127.0.0.1", "::1"} MAX_HTTP_BODY_BYTES = 1_000_000 @@ -17,7 +18,7 @@ class McpHttpServer(ThreadingHTTPServer): def __init__(self, server_address: tuple[str, int], handler: type[BaseHTTPRequestHandler]) -> None: super().__init__(server_address, handler) - self.mcp_server: McpGraphServer + self.mcp_runtime: GraphRuntimeConfig self.endpoint_path: str @@ -35,14 +36,14 @@ def build_http_server( if not allow_remote and host not in LOCAL_ORIGINS: log_event("mcp.http_remote_bind_rejected", level="WARNING", host=host, port=port) raise ValueError("MCP HTTP transport may only bind to localhost unless allow_remote is enabled") - graph_server = McpGraphServer.from_paths( + graph_runtime = runtime_config( repo_root=repo_root, config_path=config_path, db_path=db_path, manifest_path=manifest_path, ) httpd = McpHttpServer((host, port), _McpHttpHandler) - httpd.mcp_server = graph_server + httpd.mcp_runtime = graph_runtime httpd.endpoint_path = endpoint_path return httpd @@ -94,7 +95,7 @@ def do_POST(self) -> None: if not isinstance(message, dict): self._send_json(rpc_error(None, -32600, "JSON-RPC payload must be an object"), status=HTTPStatus.BAD_REQUEST) return - response = self.server.mcp_server.handle_json_rpc(message) + response = McpGraphServer(self.server.mcp_runtime).handle_json_rpc(message) if response is None: self.send_response(HTTPStatus.ACCEPTED) self.end_headers() diff --git a/src/codebase_graph/setup/instructions.py b/src/codebase_graph/setup/instructions.py index a62fadd..183ad33 100644 --- a/src/codebase_graph/setup/instructions.py +++ b/src/codebase_graph/setup/instructions.py @@ -36,6 +36,12 @@ def upsert_instruction_block( return InstructionResult(action, path.as_posix()) +def instruction_target_path(repo_root: Path, *, target: str = "auto") -> Path | None: + if target == "skip": + return None + return _select_instruction_path(repo_root, target) + + def remove_instruction_block(path: Path) -> bool: if not path.exists(): return False diff --git a/src/codebase_graph/setup/orchestrator.py b/src/codebase_graph/setup/orchestrator.py index bc9deb4..59bf6a5 100644 --- a/src/codebase_graph/setup/orchestrator.py +++ b/src/codebase_graph/setup/orchestrator.py @@ -1,13 +1,14 @@ from __future__ import annotations -from dataclasses import dataclass +import shutil +from dataclasses import dataclass, field from pathlib import Path from typing import Any from codebase_graph.diagnostics import log_event from codebase_graph.ingest import GraphMaterializer -from .instructions import InstructionResult, upsert_instruction_block +from .instructions import InstructionResult, instruction_target_path, upsert_instruction_block from .mcp_config import McpConfigResult, configure_mcp_client, server_entry from .preflight import validate_ladybug_runtime from .state import MCP_SERVER_NAME, SetupPaths, build_setup_config, derive_setup_paths, write_setup_config @@ -59,35 +60,66 @@ def run_setup(options: SetupOptions) -> SetupResult: ) paths = derive_setup_paths(options.repo_root) validate_ladybug_runtime() - paths.state_dir.mkdir(parents=True, exist_ok=True) mcp_entry = server_entry(paths.config_path) config_payload = build_setup_config(paths, mcp_command=[mcp_entry["command"], *mcp_entry["args"]]) - config_action = write_setup_config(paths.config_path, config_payload) - instructions = upsert_instruction_block( - paths.repo_root, - target=options.instructions_target, - server_name=MCP_SERVER_NAME, - config_path=paths.config_path, - setup_command=mcp_entry["command"], - ) - materializer = GraphMaterializer( - paths.repo_root, - db_path=paths.db_path, - manifest_path=paths.manifest_path, - include_fts=True, - repository_label=paths.repo_name, - ) - try: - materialization = materializer.materialize(mode=options.mode) # type: ignore[arg-type] - finally: - materializer.close() - mcp_result = configure_mcp_client( - client=options.mcp_client, - config_path=options.mcp_config_path, - setup_config_path=paths.config_path, - dry_run=options.dry_run, - skip=options.skip_mcp_config, - ) + if options.dry_run: + materialization = _dry_run_materialization(paths) + config_action = "dry_run" if _config_would_change(paths.config_path, config_payload) else "unchanged" + target_path = instruction_target_path(paths.repo_root, target=options.instructions_target) + instructions = InstructionResult("dry_run" if target_path is not None else "skipped", _path_text(target_path)) + else: + target_path = instruction_target_path(paths.repo_root, target=options.instructions_target) + previous_config = _snapshot_file(paths.config_path) + previous_instructions = _snapshot_file(target_path) + state_dir_existed = paths.state_dir.exists() + materializer = GraphMaterializer( + paths.repo_root, + db_path=paths.db_path, + manifest_path=paths.manifest_path, + include_fts=True, + repository_label=paths.repo_name, + ) + try: + config_action = write_setup_config(paths.config_path, config_payload) + instructions = upsert_instruction_block( + paths.repo_root, + target=options.instructions_target, + server_name=MCP_SERVER_NAME, + config_path=paths.config_path, + setup_command=mcp_entry["command"], + ) + materialization = materializer.materialize(mode=options.mode) # type: ignore[arg-type] + mcp_result = configure_mcp_client( + client=options.mcp_client, + config_path=options.mcp_config_path, + setup_config_path=paths.config_path, + dry_run=False, + skip=options.skip_mcp_config, + ) + except Exception: + _restore_file(paths.config_path, previous_config) + _restore_file(target_path, previous_instructions) + if not state_dir_existed: + shutil.rmtree(paths.state_dir, ignore_errors=True) + raise + finally: + materializer.close() + if options.dry_run and not options.skip_mcp_config: + mcp_result = configure_mcp_client( + client=options.mcp_client, + config_path=options.mcp_config_path, + setup_config_path=paths.config_path, + dry_run=True, + skip=False, + ) + elif options.dry_run: + mcp_result = configure_mcp_client( + client=options.mcp_client, + config_path=options.mcp_config_path, + setup_config_path=paths.config_path, + dry_run=True, + skip=True, + ) except Exception as exc: log_event( "setup.failed", @@ -132,3 +164,75 @@ def _materialization_payload(result: Any) -> dict[str, Any]: "deleted_paths": list(getattr(result, "deleted_paths")), "graph_summary": dict(getattr(result, "graph_summary")), } + + +def _dry_run_materialization(paths: SetupPaths) -> Any: + materializer = GraphMaterializer( + paths.repo_root, + db_path=paths.db_path, + manifest_path=paths.manifest_path, + include_fts=True, + repository_label=paths.repo_name, + ) + try: + snapshots, diagnostics = materializer._scan_source_files() + finally: + materializer.close() + skipped_paths = tuple(sorted(path for path, snapshot in snapshots.items() if snapshot.language is None)) + return _DryRunMaterialization( + scanned=len(snapshots), + skipped=len(skipped_paths), + diagnostics=tuple(diagnostics), + manifest_path=paths.manifest_path.as_posix(), + skipped_paths=skipped_paths, + ) + + +@dataclass(frozen=True, slots=True) +class _DryRunMaterialization: + scanned: int + skipped: int + diagnostics: tuple[str, ...] + manifest_path: str + skipped_paths: tuple[str, ...] + mode: str = "dry_run" + rebuilt: int = 0 + deleted: int = 0 + rebuilt_paths: tuple[str, ...] = () + deleted_paths: tuple[str, ...] = () + graph_summary: dict[str, Any] = field(default_factory=dict) + + +def _config_would_change(path: Path, payload: dict[str, Any]) -> bool: + if not path.exists(): + return True + try: + import json + + with path.open("r", encoding="utf-8") as handle: + return json.load(handle) != payload + except Exception: + return True + + +def _path_text(path: Path | None) -> str | None: + return path.as_posix() if path is not None else None + + +def _snapshot_file(path: Path | None) -> str | None: + if path is None or not path.exists(): + return None + return path.read_text(encoding="utf-8") + + +def _restore_file(path: Path | None, previous: str | None) -> None: + if path is None: + return + if previous is None: + try: + path.unlink() + except FileNotFoundError: + return + return + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(previous, encoding="utf-8") diff --git a/src/codebase_graph/setup/state.py b/src/codebase_graph/setup/state.py index 1cac1d5..7aecd7d 100644 --- a/src/codebase_graph/setup/state.py +++ b/src/codebase_graph/setup/state.py @@ -24,6 +24,10 @@ def derive_setup_paths(repo_root: str | Path) -> SetupPaths: raise FileNotFoundError(f"Repository root does not exist: {paths.repo_root}") if not paths.repo_root.is_dir(): raise NotADirectoryError(f"Repository root is not a directory: {paths.repo_root}") + if DEFAULT_STATE_DIR in paths.repo_root.parts: + raise ValueError( + f"Repository root may not be inside a {DEFAULT_STATE_DIR} state directory: {paths.repo_root}" + ) return paths diff --git a/tests/test_materializer.py b/tests/test_materializer.py index 851e4e6..30f0a9b 100644 --- a/tests/test_materializer.py +++ b/tests/test_materializer.py @@ -111,7 +111,7 @@ def test_scan_source_files_prunes_excluded_directories(tmp_path: Path, monkeypat observed_dirnames: list[tuple[str, ...]] = [] def fake_walk(root: Path) -> object: - dirnames = [".venv", "src", "package.egg-info"] + dirnames = [".mypy_cache", ".tox", ".venv", "node_modules", "src", "package.egg-info", "vendor"] yield Path(root).as_posix(), dirnames, [] observed_dirnames.append(tuple(dirnames)) for dirname in dirnames: @@ -127,6 +127,35 @@ def fake_walk(root: Path) -> object: assert not diagnostics +def test_scan_source_files_does_not_hash_unsupported_files( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + source_root = tmp_path / "project" + source_root.mkdir() + unsupported = source_root / "archive.bin" + supported = source_root / "service.py" + unsupported.write_bytes(b"\0" * 1024) + supported.write_text("VALUE = 1\n", encoding="utf-8") + hashed_paths: list[str] = [] + real_file_hash = materializer_module._file_hash + + def recording_file_hash(path: Path) -> str: + hashed_paths.append(path.name) + return real_file_hash(path) + + monkeypatch.setattr(materializer_module, "_file_hash", recording_file_hash) + materializer = GraphMaterializer(source_root, db_path=":memory:", manifest_path=tmp_path / "manifest.json", store=object()) + + snapshots, diagnostics = materializer._scan_source_files() + + assert hashed_paths == ["service.py"] + assert snapshots["archive.bin"].language is None + assert snapshots["archive.bin"].content_hash == "" + assert snapshots["service.py"].content_hash + assert diagnostics == ["Skipped unsupported file: archive.bin"] + + def test_full_materialization_writes_python_graph_to_ladybug(tmp_path: Path) -> None: pytest.importorskip("tree_sitter") pytest.importorskip("tree_sitter_python") diff --git a/tests/test_release_workflows.py b/tests/test_release_workflows.py new file mode 100644 index 0000000..2e71479 --- /dev/null +++ b/tests/test_release_workflows.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import re +from pathlib import Path + + +WORKFLOWS = ( + Path(".github/workflows/ci.yml"), + Path(".github/workflows/release.yml"), +) + + +def test_github_actions_are_pinned_to_immutable_commits() -> None: + mutable_refs: list[str] = [] + uses_pattern = re.compile(r"^\s*uses:\s*(?P[^@\s]+)@(?P[0-9a-f]{40}|[^\s#]+)") + + for path in WORKFLOWS: + for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): + match = uses_pattern.match(line) + if match is None: + continue + if not re.fullmatch(r"[0-9a-f]{40}", match.group("ref")): + mutable_refs.append(f"{path}:{line_number}: {match.group('action')}@{match.group('ref')}") + + assert mutable_refs == [] + + +def test_release_workflows_smoke_test_wheel_and_sdist() -> None: + for path in WORKFLOWS: + text = path.read_text(encoding="utf-8") + assert "pip install dist/*.whl" in text + assert "pip install dist/*.tar.gz" in text + + +def test_hosted_workflows_run_real_vulnerability_scans() -> None: + for path in WORKFLOWS: + text = path.read_text(encoding="utf-8") + assert "pip_audit --strict" in text + assert "pip_audit --strict --dry-run" not in text diff --git a/tests/test_search.py b/tests/test_search.py index 3635770..ebf4641 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -628,6 +628,13 @@ def test_graph_query_rejects_unbounded_response_limits() -> None: _query_payload(store, {"statement": "MATCH (n) RETURN n", "limit": MAX_GRAPH_QUERY_LIMIT + 1}) +def test_graph_query_rejects_procedure_calls() -> None: + store = _RecordingStore([[1]]) + + with pytest.raises(ValueError, match="blocked keyword: CALL"): + _query_payload(store, {"statement": "CALL CREATE_FTS_INDEX('File', 'label')"}) + + def _require_graph_runtime() -> None: pytest.importorskip("tree_sitter") pytest.importorskip("tree_sitter_python") diff --git a/tests/test_setup_workflow.py b/tests/test_setup_workflow.py index 16a9499..b2eda0b 100644 --- a/tests/test_setup_workflow.py +++ b/tests/test_setup_workflow.py @@ -165,6 +165,67 @@ def fail_preflight() -> None: assert not (repo_root / ".codebaseGraph").exists() +def test_setup_rejects_state_directory_as_repo_root(tmp_path: Path) -> None: + state_root = tmp_path / ".codebaseGraph" + state_root.mkdir() + + with pytest.raises(SetupError, match="state directory"): + run_setup(SetupOptions(repo_root=state_root, mcp_client="none")) + + assert not (state_root / ".codebaseGraph").exists() + + +def test_setup_dry_run_does_not_write_repo_or_client_state( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + pytest.importorskip("real_ladybug") + repo_root = _fresh_repo(tmp_path) + mcp_config_path = tmp_path / "config.toml" + + exit_code = cli_main( + [ + "setup", + "--repo-root", + repo_root.as_posix(), + "--mcp-client", + "codex", + "--mcp-config-path", + mcp_config_path.as_posix(), + "--dry-run", + ] + ) + payload = json.loads(capsys.readouterr().out) + + assert exit_code == 0 + assert payload["config_action"] == "dry_run" + assert payload["materialization"]["mode"] == "dry_run" + assert payload["instructions"]["action"] == "dry_run" + assert payload["mcp_config"]["action"] == "dry_run" + assert not (repo_root / ".codebaseGraph").exists() + assert not (repo_root / "AGENTS.md").exists() + assert not mcp_config_path.exists() + + +def test_setup_materialization_failure_rolls_back_published_control_files( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + pytest.importorskip("real_ladybug") + repo_root = _fresh_repo(tmp_path) + + def fail_materialize(self: object, *, mode: str = "changed") -> object: + raise RuntimeError("materialization failed") + + monkeypatch.setattr("codebase_graph.setup.orchestrator.GraphMaterializer.materialize", fail_materialize) + + with pytest.raises(SetupError, match="materialization failed"): + run_setup(SetupOptions(repo_root=repo_root, mcp_client="none")) + + assert not (repo_root / ".codebaseGraph").exists() + assert not (repo_root / "AGENTS.md").exists() + + def test_mcp_graph_query_rejects_write_like_statements(tmp_path: Path) -> None: pytest.importorskip("tree_sitter") pytest.importorskip("tree_sitter_python") From deab1635a55fef510f5e3af685488d2fa574337f Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Wed, 27 May 2026 16:17:17 +0930 Subject: [PATCH 28/53] chore: add security release guardrails The repository now has a security reporting policy, release-gate documentation references it, CI and release jobs have explicit timeouts, and tests enforce workflow timeout, advisory-scan, action-pin, wheel/sdist smoke, and security-policy expectations. The sdist manifest explicitly includes SECURITY.md so published source artifacts carry the reporting policy. Constraint: SPDX license selection, PyPI Trusted Publisher settings, and conda-forge scope remain owner-controlled production gates. Rejected: Rely on setuptools-scm file discovery for the new security policy | the staged build omitted SECURITY.md from the source distribution until MANIFEST.in made it explicit. Rejected: Add PyYAML just to inspect workflow tests | a dependency-free text guard covers the workflow timeout invariant. Confidence: high Scope-risk: narrow Directive: Keep SECURITY.md in the source distribution whenever package artifact contents change. Directive: Do not remove workflow timeouts without replacing them with an equivalent CI hang guard. Tested: ./.venv/bin/python -m pytest Tested: ./.venv/bin/ruff check . Tested: ruby YAML parse for CI and release workflows Tested: ./.venv/bin/python -m build --no-isolation --outdir /private/tmp/codebasegraph-dist-prod-ready-security-2 Tested: ./.venv/bin/python -m twine check /private/tmp/codebasegraph-dist-prod-ready-security-2/* Tested: tar listing confirms SECURITY.md and MANIFEST.in are included in the sdist Tested: ./.venv/bin/python -m pip check Tested: codebase-graph setup refresh and graph-query count Not-tested: Hosted GitHub CI execution, GitHub private vulnerability reporting settings, SPDX license metadata, PyPI Trusted Publisher setup. --- .github/workflows/ci.yml | 4 +++ .github/workflows/release.yml | 3 +++ MANIFEST.in | 1 + README.md | 4 +++ SECURITY.md | 30 +++++++++++++++++++++ docs/release.md | 1 + tests/test_release_workflows.py | 46 ++++++++++++++++++++++++++++++++- 7 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 MANIFEST.in create mode 100644 SECURITY.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4d5b7ae..d5b37eb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,6 +17,7 @@ jobs: test: name: pytest (${{ matrix.os }}, py${{ matrix.python-version }}) runs-on: ${{ matrix.os }} + timeout-minutes: 30 strategy: fail-fast: false matrix: @@ -54,6 +55,7 @@ jobs: lint: name: ruff runs-on: ubuntu-latest + timeout-minutes: 10 steps: - name: Check out repository @@ -78,6 +80,7 @@ jobs: supply-chain: name: supply chain runs-on: ubuntu-latest + timeout-minutes: 20 steps: - name: Check out repository @@ -122,6 +125,7 @@ jobs: package: name: package runs-on: ubuntu-latest + timeout-minutes: 20 steps: - name: Check out repository diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8a1234a..fa6f2ac 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,6 +13,7 @@ jobs: release-please: name: release please runs-on: ubuntu-latest + timeout-minutes: 10 permissions: contents: write pull-requests: write @@ -35,6 +36,7 @@ jobs: needs: release-please if: needs.release-please.outputs.release-created == 'true' runs-on: ubuntu-latest + timeout-minutes: 30 permissions: contents: write outputs: @@ -157,6 +159,7 @@ jobs: - build if: needs.release-please.outputs.release-created == 'true' runs-on: ubuntu-latest + timeout-minutes: 10 environment: name: pypi url: https://pypi.org/p/codebase-graph diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..bf54557 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include SECURITY.md diff --git a/README.md b/README.md index ac88199..7796f03 100644 --- a/README.md +++ b/README.md @@ -219,6 +219,10 @@ GitHub Actions runs pytest across Linux, macOS, and Windows for Python 3.10 thro Conda distribution uses the conda-forge staged-recipes path rather than direct Anaconda.org uploads. See [docs/release.md](docs/release.md) for the release workflow and conda-forge submission checklist. +## Security + +Report suspected vulnerabilities privately. See [SECURITY.md](SECURITY.md) for supported versions, reporting expectations, and the local-first MCP security boundary. + ## Troubleshooting - Missing LadyBugDB: install a package build that includes `real_ladybug`; setup will fail before creating `.codebaseGraph`. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..fddf539 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,30 @@ +# Security Policy + +## Supported Versions + +Security fixes are prepared against the current `main` branch and included in the next package release. + +## Reporting a Vulnerability + +Report suspected vulnerabilities privately through GitHub security advisories or private vulnerability reporting for this repository. Do not open a public issue for exploitable behavior, dependency vulnerabilities with an available proof of concept, credential exposure, or a bypass of the read-only MCP/query contract. + +Include: + +- affected version or commit +- reproduction steps +- expected impact +- relevant logs or proof of concept +- whether the report can be disclosed publicly after a fix is available + +Maintainers should acknowledge reports within 7 days, triage severity, and coordinate disclosure timing with the reporter. + +## Security Scope + +The production security boundary is local-first: + +- The stdio MCP transport is intended for local MCP clients. +- The HTTP MCP transport binds to localhost by default. +- `--allow-remote` only permits a non-local bind address; it does not add authentication, TLS, rate limiting, authorization, or a multi-user session model. +- `graph_query` is intended to remain read-only. Do not relax query restrictions without a parser-level read-only proof or an explicit safe-procedure allowlist. + +Dependency vulnerability scanning runs in hosted CI and release workflows. Local setup commands must not call external advisory services implicitly. diff --git a/docs/release.md b/docs/release.md index 40c8cda..53d9ac9 100644 --- a/docs/release.md +++ b/docs/release.md @@ -37,6 +37,7 @@ Pull requests and pushes to `main` or `codex/**` run: Before publishing a production release, confirm: - Hosted CI is green for tests, ruff, package build, supply-chain, wheel smoke, and source-distribution smoke. +- `SECURITY.md` is present and vulnerability reporting expectations are current. - The project owner has selected an SPDX license, added package license metadata, and included the corresponding license file. - The PyPI Trusted Publisher, `pypi` GitHub environment, and release-please token posture have been verified in GitHub/PyPI settings. - Conda-forge submission is either explicitly out of scope for the release or the recipe placeholders have been replaced with the release version, source-distribution SHA256, and chosen SPDX license. diff --git a/tests/test_release_workflows.py b/tests/test_release_workflows.py index 2e71479..b740bdc 100644 --- a/tests/test_release_workflows.py +++ b/tests/test_release_workflows.py @@ -36,4 +36,48 @@ def test_hosted_workflows_run_real_vulnerability_scans() -> None: for path in WORKFLOWS: text = path.read_text(encoding="utf-8") assert "pip_audit --strict" in text - assert "pip_audit --strict --dry-run" not in text + assert re.search(r"pip_audit\b[^\n]*--dry-run", text) is None + + +def test_security_policy_exists() -> None: + text = Path("SECURITY.md").read_text(encoding="utf-8") + + assert "Reporting a Vulnerability" in text + assert "graph_query" in text + assert "--allow-remote" in text + + +def test_workflow_jobs_have_timeouts() -> None: + for path in WORKFLOWS: + missing = _jobs_missing_timeout(path.read_text(encoding="utf-8")) + + assert missing == [] + + +def _jobs_missing_timeout(text: str) -> list[str]: + missing: list[str] = [] + in_jobs = False + current_job: str | None = None + current_has_timeout = False + + for line in text.splitlines(): + if line == "jobs:": + in_jobs = True + continue + if not in_jobs: + continue + if line and not line.startswith(" "): + break + job_match = re.match(r"^ ([A-Za-z0-9_-]+):\s*$", line) + if job_match is not None: + if current_job is not None and not current_has_timeout: + missing.append(current_job) + current_job = job_match.group(1) + current_has_timeout = False + continue + if current_job is not None and re.match(r"^ timeout-minutes:\s*\d+\s*$", line): + current_has_timeout = True + + if current_job is not None and not current_has_timeout: + missing.append(current_job) + return missing From 3a62ef2a1adec7205a39d727466cbfebf7a45b69 Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Wed, 27 May 2026 16:24:39 +0930 Subject: [PATCH 29/53] chore: add release gate checker Adds a stdlib release gate checker that passes local repository gates and reports owner/external production blockers explicitly. Documentation now shows local and production modes, and MANIFEST.in packages the checker with SECURITY.md. Constraint: SPDX license selection, PyPI Trusted Publisher setup, hosted CI status, private vulnerability reporting settings, and conda release scope are owner/external production gates. Rejected: Treat documentation alone as the release gate | it cannot be executed or checked before publishing. Rejected: Add PyYAML for workflow inspection | the checker keeps release-gate validation stdlib-only. Confidence: high Scope-risk: narrow Directive: Keep local release-gate mode passing and production mode explicit about unresolved owner/external gates. Tested: ./.venv/bin/python -m pytest Tested: ./.venv/bin/ruff check . Tested: ./.venv/bin/python scripts/check_release_gate.py Tested: ./.venv/bin/python scripts/check_release_gate.py --production --require-conda Tested: ./.venv/bin/python -m build --no-isolation --outdir /private/tmp/codebasegraph-dist-release-gate Tested: ./.venv/bin/python -m twine check /private/tmp/codebasegraph-dist-release-gate/* Tested: sdist tar listing confirms SECURITY.md, MANIFEST.in, and scripts/check_release_gate.py are packaged Tested: ./.venv/bin/python -m pip check Tested: codebase-graph setup refresh and graph-query count Not-tested: Hosted GitHub CI execution, PyPI Trusted Publisher settings, GitHub private vulnerability reporting settings, project license selection, conda-forge submission. --- MANIFEST.in | 1 + README.md | 2 + docs/release.md | 13 +++ scripts/check_release_gate.py | 200 ++++++++++++++++++++++++++++++++ tests/test_release_workflows.py | 41 +++---- 5 files changed, 230 insertions(+), 27 deletions(-) create mode 100644 scripts/check_release_gate.py diff --git a/MANIFEST.in b/MANIFEST.in index bf54557..10b457c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1,2 @@ include SECURITY.md +include scripts/check_release_gate.py diff --git a/README.md b/README.md index 7796f03..2fd1528 100644 --- a/README.md +++ b/README.md @@ -217,6 +217,8 @@ ruff check . GitHub Actions runs pytest across Linux, macOS, and Windows for Python 3.10 through 3.14, plus ruff, supply-chain, and package-build validation. Supply-chain checks include dependency consistency, vulnerability advisory scanning, Dependabot update coverage, immutable GitHub Action pins, and CycloneDX SBOM generation. Built wheels and source distributions are smoke-tested with `setup`, `graph-health`, `graph-search`, and a stdio MCP handshake before release. Releases are managed by release-please, use tag-derived package versions, create GitHub Releases with distribution assets and SBOMs, and publish to PyPI through Trusted Publishing. +Run `python scripts/check_release_gate.py` for local release-gate checks. Use the `--production` confirmations documented in [docs/release.md](docs/release.md) before publishing. + Conda distribution uses the conda-forge staged-recipes path rather than direct Anaconda.org uploads. See [docs/release.md](docs/release.md) for the release workflow and conda-forge submission checklist. ## Security diff --git a/docs/release.md b/docs/release.md index 53d9ac9..403dc13 100644 --- a/docs/release.md +++ b/docs/release.md @@ -42,6 +42,19 @@ Before publishing a production release, confirm: - The PyPI Trusted Publisher, `pypi` GitHub environment, and release-please token posture have been verified in GitHub/PyPI settings. - Conda-forge submission is either explicitly out of scope for the release or the recipe placeholders have been replaced with the release version, source-distribution SHA256, and chosen SPDX license. +Run the local release-gate checker before publishing: + +```bash +python scripts/check_release_gate.py +python scripts/check_release_gate.py --production \ + --confirm trusted-publisher \ + --confirm pypi-environment \ + --confirm hosted-ci-green \ + --confirm private-vulnerability-reporting +``` + +Add `--require-conda` when conda-forge submission is in scope for the release. + Vulnerability advisory scans require an external advisory service. Hosted CI and release workflows run those scans and fail on known vulnerable dependencies. Local setup stays offline-safe and must not call external advisory APIs implicitly; run local advisory scans explicitly when that disclosure is acceptable. diff --git a/scripts/check_release_gate.py b/scripts/check_release_gate.py new file mode 100644 index 0000000..661f59a --- /dev/null +++ b/scripts/check_release_gate.py @@ -0,0 +1,200 @@ +from __future__ import annotations + +import argparse +import re +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +try: + import tomllib +except ImportError: # pragma: no cover - Python 3.10 compatibility + import tomli as tomllib + + +REPO_ROOT = Path(__file__).resolve().parents[1] +WORKFLOWS = ( + Path(".github/workflows/ci.yml"), + Path(".github/workflows/release.yml"), +) +PYPI_CONFIRMATION_FLAGS = ( + "trusted-publisher", + "pypi-environment", + "hosted-ci-green", + "private-vulnerability-reporting", +) + + +@dataclass(frozen=True, slots=True) +class GateIssue: + severity: str + code: str + message: str + + def line(self) -> str: + return f"{self.severity}: {self.code}: {self.message}" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Check local and production release readiness gates.") + parser.add_argument("--production", action="store_true", help="Require owner-controlled production release gates.") + parser.add_argument("--require-conda", action="store_true", help="Require the conda-forge recipe to be finalized.") + parser.add_argument( + "--confirm", + action="append", + default=[], + choices=PYPI_CONFIRMATION_FLAGS, + help="Confirm a manually verified external production gate.", + ) + args = parser.parse_args(argv) + + issues = run_checks( + production=args.production, + require_conda=args.require_conda, + confirmations=set(args.confirm), + ) + if not issues: + print("release gate passed") + return 0 + for issue in issues: + print(issue.line(), file=sys.stderr) + return 1 if any(issue.severity == "FAIL" for issue in issues) else 0 + + +def run_checks(*, production: bool, require_conda: bool, confirmations: set[str]) -> list[GateIssue]: + issues: list[GateIssue] = [] + issues.extend(_check_security_policy()) + issues.extend(_check_workflows()) + issues.extend(_check_release_workflow_permissions()) + if production: + issues.extend(_check_license_metadata()) + issues.extend(_check_external_confirmations(confirmations)) + if require_conda: + issues.extend(_check_conda_recipe()) + return issues + + +def _check_security_policy() -> list[GateIssue]: + issues: list[GateIssue] = [] + security = REPO_ROOT / "SECURITY.md" + manifest = REPO_ROOT / "MANIFEST.in" + if not security.exists(): + issues.append(GateIssue("FAIL", "security-policy-missing", "SECURITY.md is required.")) + if security.exists(): + text = security.read_text(encoding="utf-8") + for required in ("Reporting a Vulnerability", "graph_query", "--allow-remote"): + if required not in text: + issues.append(GateIssue("FAIL", "security-policy-incomplete", f"SECURITY.md must mention {required!r}.")) + if not manifest.exists() or "include SECURITY.md" not in manifest.read_text(encoding="utf-8"): + issues.append(GateIssue("FAIL", "security-policy-not-packaged", "MANIFEST.in must include SECURITY.md.")) + return issues + + +def _check_workflows() -> list[GateIssue]: + issues: list[GateIssue] = [] + for relative_path in WORKFLOWS: + path = REPO_ROOT / relative_path + text = path.read_text(encoding="utf-8") + issues.extend(_workflow_action_pin_issues(relative_path, text)) + for job in _jobs_missing_timeout(text): + issues.append(GateIssue("FAIL", "workflow-timeout-missing", f"{relative_path}:{job} has no timeout.")) + if re.search(r"pip_audit\b[^\n]*--dry-run", text): + issues.append(GateIssue("FAIL", "workflow-audit-dry-run", f"{relative_path} uses pip-audit --dry-run.")) + if "pip install dist/*.whl" not in text: + issues.append(GateIssue("FAIL", "workflow-wheel-smoke-missing", f"{relative_path} must smoke-test wheels.")) + if "pip install dist/*.tar.gz" not in text: + issues.append(GateIssue("FAIL", "workflow-sdist-smoke-missing", f"{relative_path} must smoke-test sdists.")) + return issues + + +def _workflow_action_pin_issues(relative_path: Path, text: str) -> list[GateIssue]: + issues: list[GateIssue] = [] + uses_pattern = re.compile(r"^\s*uses:\s*(?P[^@\s]+)@(?P[0-9a-f]{40}|[^\s#]+)") + for line_number, line in enumerate(text.splitlines(), start=1): + match = uses_pattern.match(line) + if match is None: + continue + if not re.fullmatch(r"[0-9a-f]{40}", match.group("ref")): + target = f"{relative_path}:{line_number}: {match.group('action')}@{match.group('ref')}" + issues.append(GateIssue("FAIL", "workflow-action-not-pinned", target)) + return issues + + +def _jobs_missing_timeout(text: str) -> list[str]: + missing: list[str] = [] + in_jobs = False + current_job: str | None = None + current_has_timeout = False + + for line in text.splitlines(): + if line == "jobs:": + in_jobs = True + continue + if not in_jobs: + continue + if line and not line.startswith(" "): + break + job_match = re.match(r"^ ([A-Za-z0-9_-]+):\s*$", line) + if job_match is not None: + if current_job is not None and not current_has_timeout: + missing.append(current_job) + current_job = job_match.group(1) + current_has_timeout = False + continue + if current_job is not None and re.match(r"^ timeout-minutes:\s*\d+\s*$", line): + current_has_timeout = True + + if current_job is not None and not current_has_timeout: + missing.append(current_job) + return missing + + +def _check_release_workflow_permissions() -> list[GateIssue]: + text = (REPO_ROOT / ".github/workflows/release.yml").read_text(encoding="utf-8") + issues: list[GateIssue] = [] + if "environment:" not in text or "name: pypi" not in text: + issues.append(GateIssue("FAIL", "pypi-environment-missing", "release workflow must publish through pypi environment.")) + if "id-token: write" not in text: + issues.append(GateIssue("FAIL", "pypi-oidc-missing", "release workflow must grant id-token: write.")) + return issues + + +def _check_license_metadata() -> list[GateIssue]: + pyproject = _load_toml(REPO_ROOT / "pyproject.toml") + project = pyproject.get("project", {}) + license_value = project.get("license") + license_files = project.get("license-files") or pyproject.get("tool", {}).get("setuptools", {}).get("license-files") + license_paths = [path for path in REPO_ROOT.iterdir() if path.name.upper().startswith("LICENSE")] + issues: list[GateIssue] = [] + if not license_value and not license_files: + issues.append(GateIssue("FAIL", "license-metadata-missing", "pyproject.toml must declare package license metadata.")) + if not license_paths: + issues.append(GateIssue("FAIL", "license-file-missing", "repository must include the selected license file.")) + return issues + + +def _check_external_confirmations(confirmations: set[str]) -> list[GateIssue]: + return [ + GateIssue("FAIL", "external-confirmation-missing", f"production release requires --confirm {flag}.") + for flag in PYPI_CONFIRMATION_FLAGS + if flag not in confirmations + ] + + +def _check_conda_recipe() -> list[GateIssue]: + recipe = (REPO_ROOT / "conda-forge/recipe/meta.yaml").read_text(encoding="utf-8") + issues: list[GateIssue] = [] + for placeholder in ("PUT_RELEASE_SDIST_SHA256_HERE", "PUT_SPDX_LICENSE_ID_HERE"): + if placeholder in recipe: + issues.append(GateIssue("FAIL", "conda-placeholder", f"conda recipe still contains {placeholder}.")) + return issues + + +def _load_toml(path: Path) -> dict[str, Any]: + with path.open("rb") as handle: + return tomllib.load(handle) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_release_workflows.py b/tests/test_release_workflows.py index b740bdc..11e42fa 100644 --- a/tests/test_release_workflows.py +++ b/tests/test_release_workflows.py @@ -3,6 +3,8 @@ import re from pathlib import Path +from scripts.check_release_gate import _jobs_missing_timeout, run_checks + WORKFLOWS = ( Path(".github/workflows/ci.yml"), @@ -54,30 +56,15 @@ def test_workflow_jobs_have_timeouts() -> None: assert missing == [] -def _jobs_missing_timeout(text: str) -> list[str]: - missing: list[str] = [] - in_jobs = False - current_job: str | None = None - current_has_timeout = False - - for line in text.splitlines(): - if line == "jobs:": - in_jobs = True - continue - if not in_jobs: - continue - if line and not line.startswith(" "): - break - job_match = re.match(r"^ ([A-Za-z0-9_-]+):\s*$", line) - if job_match is not None: - if current_job is not None and not current_has_timeout: - missing.append(current_job) - current_job = job_match.group(1) - current_has_timeout = False - continue - if current_job is not None and re.match(r"^ timeout-minutes:\s*\d+\s*$", line): - current_has_timeout = True - - if current_job is not None and not current_has_timeout: - missing.append(current_job) - return missing +def test_local_release_gate_passes() -> None: + assert run_checks(production=False, require_conda=False, confirmations=set()) == [] + + +def test_production_release_gate_reports_owner_controlled_blockers() -> None: + issues = run_checks(production=True, require_conda=True, confirmations=set()) + codes = {issue.code for issue in issues} + + assert "license-metadata-missing" in codes + assert "license-file-missing" in codes + assert "external-confirmation-missing" in codes + assert "conda-placeholder" in codes From 6a93f59b08b3a8c72e2f51fc2775ea714efd2e04 Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Wed, 27 May 2026 16:43:05 +0930 Subject: [PATCH 30/53] fix: enforce production release and HTTP auth gates Blocks release distribution builds behind the repository production gate and makes remote MCP HTTP opt-in defensible by requiring a bearer token. The release gate now also catches bare external GitHub Action references instead of only mutable refs with an @ref. Constraint: License selection, PyPI Trusted Publisher setup, hosted CI status, private vulnerability reporting, and conda release scope remain owner/external production gates. Constraint: Local HTTP stays unauthenticated for local MCP clients; remote HTTP must not expose graph tools without an explicit bearer token. Rejected: Keep production release checks only in documentation | the workflow could still publish without running the executable gate. Rejected: Treat Origin checks as remote HTTP authentication | non-browser clients can omit Origin and still exfiltrate graph data. Rejected: Add a full auth framework | bearer-token enforcement closes the remote exposure without new dependencies or a wider protocol change. Confidence: high Scope-risk: moderate Directive: Do not re-enable --allow-remote without an auth boundary and do not let release builds bypass scripts/check_release_gate.py --production. Tested: ./.venv/bin/python -m pytest Tested: ./.venv/bin/ruff check . Tested: ./.venv/bin/python scripts/check_release_gate.py Tested: ./.venv/bin/python scripts/check_release_gate.py --production --require-conda Tested: ./.venv/bin/python -m build --no-isolation --outdir /private/tmp/codebasegraph-dist-release-http-gates Tested: ./.venv/bin/python -m twine check /private/tmp/codebasegraph-dist-release-http-gates/* Tested: ./.venv/bin/python -m pip check Tested: sdist tar listing confirms release workflow, README, SECURITY.md, and scripts/check_release_gate.py are packaged Tested: codebase-graph setup refresh and graph-query count Not-tested: Hosted GitHub Actions execution, PyPI Trusted Publisher settings, GitHub private vulnerability reporting settings, project license selection, conda-forge submission, remote 0.0.0.0 socket acceptance in this sandbox. --- .github/workflows/release.yml | 51 ++++++++++++- README.md | 10 ++- SECURITY.md | 2 +- docs/release.md | 12 +++ scripts/check_release_gate.py | 32 +++++++- src/codebase_graph/cli/__init__.py | 22 +++++- src/codebase_graph/mcp/transports/http.py | 44 ++++++++++- tests/test_mcp_portability.py | 89 +++++++++++++++++++++-- tests/test_release_workflows.py | 39 +++++++--- 9 files changed, 273 insertions(+), 28 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fa6f2ac..f4259d8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -31,9 +31,58 @@ jobs: config-file: release-please-config.json manifest-file: .release-please-manifest.json + production-gate: + name: production release gate + needs: release-please + if: needs.release-please.outputs.release-created == 'true' + runs-on: ubuntu-latest + timeout-minutes: 10 + environment: + name: pypi + permissions: + contents: read + + env: + RELEASE_TAG: ${{ needs.release-please.outputs.tag-name }} + CODEBASE_GRAPH_CONFIRM_TRUSTED_PUBLISHER: ${{ vars.CODEBASE_GRAPH_CONFIRM_TRUSTED_PUBLISHER }} + CODEBASE_GRAPH_CONFIRM_PYPI_ENVIRONMENT: ${{ vars.CODEBASE_GRAPH_CONFIRM_PYPI_ENVIRONMENT }} + CODEBASE_GRAPH_CONFIRM_HOSTED_CI_GREEN: ${{ vars.CODEBASE_GRAPH_CONFIRM_HOSTED_CI_GREEN }} + CODEBASE_GRAPH_CONFIRM_PRIVATE_VULNERABILITY_REPORTING: ${{ vars.CODEBASE_GRAPH_CONFIRM_PRIVATE_VULNERABILITY_REPORTING }} + CODEBASE_GRAPH_REQUIRE_CONDA: ${{ vars.CODEBASE_GRAPH_REQUIRE_CONDA }} + + steps: + - name: Check out release tag + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + ref: ${{ env.RELEASE_TAG }} + fetch-depth: 0 + + - name: Run production release gate + shell: bash + run: | + args=(--production) + if [[ "${CODEBASE_GRAPH_CONFIRM_TRUSTED_PUBLISHER}" == "true" ]]; then + args+=(--confirm trusted-publisher) + fi + if [[ "${CODEBASE_GRAPH_CONFIRM_PYPI_ENVIRONMENT}" == "true" ]]; then + args+=(--confirm pypi-environment) + fi + if [[ "${CODEBASE_GRAPH_CONFIRM_HOSTED_CI_GREEN}" == "true" ]]; then + args+=(--confirm hosted-ci-green) + fi + if [[ "${CODEBASE_GRAPH_CONFIRM_PRIVATE_VULNERABILITY_REPORTING}" == "true" ]]; then + args+=(--confirm private-vulnerability-reporting) + fi + if [[ "${CODEBASE_GRAPH_REQUIRE_CONDA}" == "true" ]]; then + args+=(--require-conda) + fi + python scripts/check_release_gate.py "${args[@]}" + build: name: build release distributions - needs: release-please + needs: + - release-please + - production-gate if: needs.release-please.outputs.release-created == 'true' runs-on: ubuntu-latest timeout-minutes: 30 diff --git a/README.md b/README.md index 2fd1528..c92b7e5 100644 --- a/README.md +++ b/README.md @@ -145,7 +145,15 @@ codebase-graph mcp http --config .codebaseGraph/config.json --host 127.0.0.1 --p ``` The HTTP transport rejects non-local bind hosts unless `--allow-remote` is passed. Keep it bound to `127.0.0.1` -for normal use; `--allow-remote` does not add authentication, TLS, rate limiting, or a multi-user session model. +for normal use. Remote binding requires a bearer token: + +```bash +CODEBASE_GRAPH_MCP_TOKEN="$(openssl rand -hex 32)" +codebase-graph mcp http --config .codebaseGraph/config.json --host 0.0.0.0 --allow-remote --auth-token-env CODEBASE_GRAPH_MCP_TOKEN +``` + +Clients must send `Authorization: Bearer `. The token gate does not add TLS, rate limiting, authorization scopes, or +a multi-user session model; put remote HTTP behind a trusted network boundary and TLS-terminating proxy. Available MCP tools: diff --git a/SECURITY.md b/SECURITY.md index fddf539..17a3334 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -24,7 +24,7 @@ The production security boundary is local-first: - The stdio MCP transport is intended for local MCP clients. - The HTTP MCP transport binds to localhost by default. -- `--allow-remote` only permits a non-local bind address; it does not add authentication, TLS, rate limiting, authorization, or a multi-user session model. +- `--allow-remote` requires a bearer token. It does not add TLS, rate limiting, authorization scopes, or a multi-user session model. - `graph_query` is intended to remain read-only. Do not relax query restrictions without a parser-level read-only proof or an explicit safe-procedure allowlist. Dependency vulnerability scanning runs in hosted CI and release workflows. Local setup commands must not call external advisory services implicitly. diff --git a/docs/release.md b/docs/release.md index 403dc13..4f232ff 100644 --- a/docs/release.md +++ b/docs/release.md @@ -13,6 +13,18 @@ Configure a PyPI Trusted Publisher for: Create the `pypi` GitHub environment before the first release. Use required reviewers on that environment when release approval should be manual. +Set these `pypi` environment variables to `true` only after the corresponding owner-controlled gate is verified: + +- `CODEBASE_GRAPH_CONFIRM_TRUSTED_PUBLISHER` +- `CODEBASE_GRAPH_CONFIRM_PYPI_ENVIRONMENT` +- `CODEBASE_GRAPH_CONFIRM_HOSTED_CI_GREEN` +- `CODEBASE_GRAPH_CONFIRM_PRIVATE_VULNERABILITY_REPORTING` +- `CODEBASE_GRAPH_REQUIRE_CONDA`, only when conda-forge publication is part of the release + +The release workflow runs `scripts/check_release_gate.py --production` in the protected `pypi` environment before building +or publishing release distributions. If one of these variables is missing or the repository-local gates fail, the release +stops before any package is uploaded. + ## CI Pull requests and pushes to `main` or `codex/**` run: diff --git a/scripts/check_release_gate.py b/scripts/check_release_gate.py index 661f59a..8ccb1ec 100644 --- a/scripts/check_release_gate.py +++ b/scripts/check_release_gate.py @@ -95,6 +95,9 @@ def _check_workflows() -> list[GateIssue]: issues: list[GateIssue] = [] for relative_path in WORKFLOWS: path = REPO_ROOT / relative_path + if not path.exists(): + issues.append(GateIssue("FAIL", "workflow-missing", f"{relative_path} is required.")) + continue text = path.read_text(encoding="utf-8") issues.extend(_workflow_action_pin_issues(relative_path, text)) for job in _jobs_missing_timeout(text): @@ -110,14 +113,22 @@ def _check_workflows() -> list[GateIssue]: def _workflow_action_pin_issues(relative_path: Path, text: str) -> list[GateIssue]: issues: list[GateIssue] = [] - uses_pattern = re.compile(r"^\s*uses:\s*(?P[^@\s]+)@(?P[0-9a-f]{40}|[^\s#]+)") + uses_pattern = re.compile(r"^\s*(?:-\s*)?uses:\s*(?P[^\s#]+)") for line_number, line in enumerate(text.splitlines(), start=1): match = uses_pattern.match(line) if match is None: continue - if not re.fullmatch(r"[0-9a-f]{40}", match.group("ref")): - target = f"{relative_path}:{line_number}: {match.group('action')}@{match.group('ref')}" - issues.append(GateIssue("FAIL", "workflow-action-not-pinned", target)) + target = match.group("target").strip("'\"") + if target.startswith(("./", "../")): + continue + if "@" not in target: + issues.append(GateIssue("FAIL", "workflow-action-not-pinned", f"{relative_path}:{line_number}: {target}")) + continue + action, ref = target.rsplit("@", 1) + if not re.fullmatch(r"[0-9a-fA-F]{40}", ref): + issues.append( + GateIssue("FAIL", "workflow-action-not-pinned", f"{relative_path}:{line_number}: {action}@{ref}") + ) return issues @@ -153,6 +164,19 @@ def _jobs_missing_timeout(text: str) -> list[str]: def _check_release_workflow_permissions() -> list[GateIssue]: text = (REPO_ROOT / ".github/workflows/release.yml").read_text(encoding="utf-8") issues: list[GateIssue] = [] + if ( + "production-gate:" not in text + or "python scripts/check_release_gate.py" not in text + or "--production" not in text + or "- production-gate" not in text + ): + issues.append( + GateIssue( + "FAIL", + "release-production-gate-missing", + "release workflow must run the production release gate before build/publish.", + ) + ) if "environment:" not in text or "name: pypi" not in text: issues.append(GateIssue("FAIL", "pypi-environment-missing", "release workflow must publish through pypi environment.")) if "id-token: write" not in text: diff --git a/src/codebase_graph/cli/__init__.py b/src/codebase_graph/cli/__init__.py index c138e83..3a52241 100644 --- a/src/codebase_graph/cli/__init__.py +++ b/src/codebase_graph/cli/__init__.py @@ -2,6 +2,7 @@ import argparse import json +import os from collections.abc import Sequence from pathlib import Path @@ -117,8 +118,14 @@ def main(argv: Sequence[str] | None = None) -> int: http_parser.add_argument( "--allow-remote", action="store_true", - help="Allow binding MCP HTTP to a non-local host; no authentication is provided", + help="Allow binding MCP HTTP to a non-local host; requires an auth token", ) + http_parser.add_argument( + "--auth-token", + default=None, + help="Bearer token required for HTTP requests; prefer --auth-token-env to avoid shell history exposure", + ) + http_parser.add_argument("--auth-token-env", default=None, help="Environment variable containing the HTTP bearer token") args = parser.parse_args(argv) if args.command == "materialize": @@ -259,6 +266,7 @@ def main(argv: Sequence[str] | None = None) -> int: if args.command == "mcp" and args.mcp_command == "http": from codebase_graph.mcp.server import serve_http + auth_token = _http_auth_token(args, parser) serve_http( repo_root=args.repo_root, config_path=args.config, @@ -268,6 +276,7 @@ def main(argv: Sequence[str] | None = None) -> int: port=args.port, endpoint_path=args.path, allow_remote=args.allow_remote, + auth_token=auth_token, ) return 0 parser.error(f"Unknown command: {args.command}") @@ -385,4 +394,15 @@ def _print_mcp_install_results(results: Sequence[object]) -> None: print(f"{client}: {action} {server_name} via {method}{suffix}") +def _http_auth_token(args: argparse.Namespace, parser: argparse.ArgumentParser) -> str | None: + if args.auth_token and args.auth_token_env: + parser.error("mcp http accepts either --auth-token or --auth-token-env, not both") + if args.auth_token_env: + value = os.environ.get(args.auth_token_env) + if not value: + parser.error(f"Environment variable {args.auth_token_env!r} must contain the HTTP bearer token") + return value + return args.auth_token + + __all__ = ["main"] diff --git a/src/codebase_graph/mcp/transports/http.py b/src/codebase_graph/mcp/transports/http.py index 50924f0..e06587c 100644 --- a/src/codebase_graph/mcp/transports/http.py +++ b/src/codebase_graph/mcp/transports/http.py @@ -1,5 +1,6 @@ from __future__ import annotations +import secrets import json from http import HTTPStatus from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer @@ -20,6 +21,7 @@ def __init__(self, server_address: tuple[str, int], handler: type[BaseHTTPReques super().__init__(server_address, handler) self.mcp_runtime: GraphRuntimeConfig self.endpoint_path: str + self.auth_token: str | None def build_http_server( @@ -32,7 +34,13 @@ def build_http_server( port: int = 8765, endpoint_path: str = "/mcp", allow_remote: bool = False, + auth_token: str | None = None, ) -> McpHttpServer: + if auth_token is not None and not auth_token.strip(): + raise ValueError("MCP HTTP auth token must not be blank") + if allow_remote and auth_token is None: + log_event("mcp.http_remote_bind_rejected", level="WARNING", host=host, port=port) + raise ValueError("MCP HTTP remote bind requires an auth token") if not allow_remote and host not in LOCAL_ORIGINS: log_event("mcp.http_remote_bind_rejected", level="WARNING", host=host, port=port) raise ValueError("MCP HTTP transport may only bind to localhost unless allow_remote is enabled") @@ -45,6 +53,7 @@ def build_http_server( httpd = McpHttpServer((host, port), _McpHttpHandler) httpd.mcp_runtime = graph_runtime httpd.endpoint_path = endpoint_path + httpd.auth_token = auth_token return httpd @@ -58,6 +67,7 @@ def serve_http( port: int = 8765, endpoint_path: str = "/mcp", allow_remote: bool = False, + auth_token: str | None = None, ) -> None: server = build_http_server( repo_root=repo_root, @@ -68,6 +78,7 @@ def serve_http( port=port, endpoint_path=endpoint_path, allow_remote=allow_remote, + auth_token=auth_token, ) try: server.serve_forever() @@ -79,7 +90,7 @@ class _McpHttpHandler(BaseHTTPRequestHandler): server: McpHttpServer def do_POST(self) -> None: - if not self._request_path_matches() or not self._valid_origin(): + if not self._request_path_matches() or not self._valid_origin() or not self._valid_auth(): return if not self._valid_protocol_header(): return @@ -103,7 +114,7 @@ def do_POST(self) -> None: self._send_json(response) def do_GET(self) -> None: - if not self._request_path_matches() or not self._valid_origin(): + if not self._request_path_matches() or not self._valid_origin() or not self._valid_auth(): return self.send_response(HTTPStatus.METHOD_NOT_ALLOWED) self.send_header("Allow", "POST") @@ -134,6 +145,25 @@ def _valid_origin(self) -> bool: self._send_json(rpc_error(None, -32000, "Forbidden origin"), status=HTTPStatus.FORBIDDEN) return False + def _valid_auth(self) -> bool: + if self.server.auth_token is None: + return True + authorization = self.headers.get("Authorization", "") + prefix = "Bearer " + if authorization.startswith(prefix) and secrets.compare_digest(authorization[len(prefix) :], self.server.auth_token): + return True + log_event( + "mcp.http_unauthorized", + level="WARNING", + client_address=self.client_address[0], + ) + self._send_json( + rpc_error(None, -32000, "Unauthorized"), + status=HTTPStatus.UNAUTHORIZED, + headers={"WWW-Authenticate": "Bearer"}, + ) + return False + def _valid_protocol_header(self) -> bool: requested = self.headers.get("MCP-Protocol-Version") if requested is None: @@ -193,10 +223,18 @@ def _content_length(self) -> int | None: return None return length - def _send_json(self, payload: dict[str, Any], *, status: HTTPStatus = HTTPStatus.OK) -> None: + def _send_json( + self, + payload: dict[str, Any], + *, + status: HTTPStatus = HTTPStatus.OK, + headers: dict[str, str] | None = None, + ) -> None: body = json.dumps(payload, separators=(",", ":"), sort_keys=True).encode("utf-8") self.send_response(status) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(body))) + for name, value in (headers or {}).items(): + self.send_header(name, value) self.end_headers() self.wfile.write(body) diff --git a/tests/test_mcp_portability.py b/tests/test_mcp_portability.py index 50cb84e..e3cf095 100644 --- a/tests/test_mcp_portability.py +++ b/tests/test_mcp_portability.py @@ -220,6 +220,35 @@ def test_http_mcp_rejects_remote_bind_without_explicit_opt_in(tmp_path: Path) -> build_http_server(repo_root=tmp_path, db_path=tmp_path / "missing.ldb", host="0.0.0.0", port=0) +def test_http_mcp_rejects_remote_bind_without_auth_token(tmp_path: Path) -> None: + with pytest.raises(ValueError, match="auth token"): + build_http_server( + repo_root=tmp_path, + db_path=tmp_path / "missing.ldb", + host="0.0.0.0", + port=0, + allow_remote=True, + ) + + +def test_http_mcp_accepts_remote_bind_with_auth_token(tmp_path: Path) -> None: + db_path = tmp_path / "graph.ldb" + db_path.write_text("", encoding="utf-8") + try: + httpd = build_http_server( + repo_root=tmp_path, + db_path=db_path, + host="0.0.0.0", + port=0, + allow_remote=True, + auth_token="secret-token", + ) + except PermissionError as exc: + pytest.skip(f"remote socket bind is unavailable in this environment: {exc}") + + httpd.server_close() + + def test_http_mcp_transport_handles_initialize_list_and_call(tmp_path: Path) -> None: pytest.importorskip("tree_sitter") pytest.importorskip("tree_sitter_python") @@ -250,6 +279,49 @@ def test_http_mcp_transport_handles_initialize_list_and_call(tmp_path: Path) -> assert exc_info.value.code == 400 +def test_http_mcp_transport_enforces_bearer_token_when_configured(tmp_path: Path) -> None: + pytest.importorskip("tree_sitter") + pytest.importorskip("tree_sitter_python") + pytest.importorskip("real_ladybug") + repo_root = _fresh_repo(tmp_path) + result = run_setup(SetupOptions(repo_root=repo_root, mcp_client="none", instructions_target="skip")) + try: + httpd = build_http_server(config_path=result.paths.config_path, host="127.0.0.1", port=0, auth_token="secret") + except PermissionError as exc: + pytest.skip(f"local socket bind is unavailable in this environment: {exc}") + thread = threading.Thread(target=httpd.serve_forever, daemon=True) + thread.start() + host, port = httpd.server_address + try: + with pytest.raises(urllib.error.HTTPError) as missing_exc: + _http_rpc(host, port, "initialize", {"protocolVersion": "2025-11-25"}, origin=f"http://{host}:{port}") + with pytest.raises(urllib.error.HTTPError) as wrong_exc: + _http_rpc( + host, + port, + "initialize", + {"protocolVersion": "2025-11-25"}, + auth_token="wrong", + origin=f"http://{host}:{port}", + ) + initialized = _http_rpc( + host, + port, + "initialize", + {"protocolVersion": "2025-11-25"}, + auth_token="secret", + origin=f"http://{host}:{port}", + ) + finally: + httpd.shutdown() + httpd.server_close() + thread.join(timeout=10) + + assert missing_exc.value.code == 401 + assert wrong_exc.value.code == 401 + assert initialized["result"]["protocolVersion"] == "2025-11-25" + + def _rpc(stdin: BinaryIO, stdout: BinaryIO, method: str, params: dict[str, Any]) -> dict[str, Any]: request_id = _rpc.counter _rpc.counter += 1 @@ -297,17 +369,22 @@ def _http_rpc( params: dict[str, Any], *, protocol_version: str = "2025-11-25", + auth_token: str | None = None, + origin: str | None = None, ) -> dict[str, Any]: payload = json.dumps({"jsonrpc": "2.0", "id": 1, "method": method, "params": params}).encode("utf-8") + headers = { + "Accept": "application/json, text/event-stream", + "Content-Type": "application/json", + "MCP-Protocol-Version": protocol_version, + "Origin": origin or f"http://{host}:{port}", + } + if auth_token is not None: + headers["Authorization"] = f"Bearer {auth_token}" request = urllib.request.Request( f"http://{host}:{port}/mcp", data=payload, - headers={ - "Accept": "application/json, text/event-stream", - "Content-Type": "application/json", - "MCP-Protocol-Version": protocol_version, - "Origin": f"http://{host}:{port}", - }, + headers=headers, method="POST", ) with urllib.request.urlopen(request, timeout=10) as response: diff --git a/tests/test_release_workflows.py b/tests/test_release_workflows.py index 11e42fa..7ebf360 100644 --- a/tests/test_release_workflows.py +++ b/tests/test_release_workflows.py @@ -3,7 +3,7 @@ import re from pathlib import Path -from scripts.check_release_gate import _jobs_missing_timeout, run_checks +from scripts.check_release_gate import _jobs_missing_timeout, _workflow_action_pin_issues, run_checks WORKFLOWS = ( @@ -13,18 +13,26 @@ def test_github_actions_are_pinned_to_immutable_commits() -> None: - mutable_refs: list[str] = [] - uses_pattern = re.compile(r"^\s*uses:\s*(?P[^@\s]+)@(?P[0-9a-f]{40}|[^\s#]+)") - for path in WORKFLOWS: - for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): - match = uses_pattern.match(line) - if match is None: - continue - if not re.fullmatch(r"[0-9a-f]{40}", match.group("ref")): - mutable_refs.append(f"{path}:{line_number}: {match.group('action')}@{match.group('ref')}") + mutable_refs = _workflow_action_pin_issues(path, path.read_text(encoding="utf-8")) + + assert mutable_refs == [] + + +def test_action_pin_checker_rejects_bare_external_actions() -> None: + text = """ +jobs: + lint: + steps: + - uses: actions/checkout + - uses: ./.github/actions/local-smoke + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 +""" - assert mutable_refs == [] + issues = _workflow_action_pin_issues(Path(".github/workflows/example.yml"), text) + + assert [issue.code for issue in issues] == ["workflow-action-not-pinned"] + assert "actions/checkout" in issues[0].message def test_release_workflows_smoke_test_wheel_and_sdist() -> None: @@ -34,6 +42,15 @@ def test_release_workflows_smoke_test_wheel_and_sdist() -> None: assert "pip install dist/*.tar.gz" in text +def test_release_workflow_enforces_production_gate_before_build() -> None: + text = Path(".github/workflows/release.yml").read_text(encoding="utf-8") + + assert "production-gate:" in text + assert "python scripts/check_release_gate.py" in text + assert "--production" in text + assert "build:\n name: build release distributions\n needs:\n - release-please\n - production-gate" in text + + def test_hosted_workflows_run_real_vulnerability_scans() -> None: for path in WORKFLOWS: text = path.read_text(encoding="utf-8") From e63f397de77d7f20fe5a51c59e4ca5ece3d80870 Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Wed, 27 May 2026 16:48:51 +0930 Subject: [PATCH 31/53] fix: recover stale materialization locks Automatically removes materialization lock files only when they contain a valid dead writer PID. Active or malformed locks still fail closed, so concurrent writers remain protected while stale crash leftovers no longer require routine manual cleanup. Constraint: Lock files are the existing cross-process writer guard for on-disk materialization. Rejected: Always delete existing lock files | that would allow concurrent writers to corrupt or race graph rebuilds. Rejected: Keep manual stale-lock deletion as the primary recovery path | production setup should recover from dead writer PIDs automatically. Confidence: high Scope-risk: narrow Directive: Treat malformed or live-PID lock files as active until inspected; only auto-remove locks with a valid non-running PID. Tested: ./.venv/bin/python -m pytest Tested: ./.venv/bin/ruff check . Tested: ./.venv/bin/python scripts/check_release_gate.py Tested: ./.venv/bin/python -m build --no-isolation --outdir /private/tmp/codebasegraph-dist-stale-lock Tested: ./.venv/bin/python -m twine check /private/tmp/codebasegraph-dist-stale-lock/* Tested: ./.venv/bin/python -m pip check Tested: sdist tar listing confirms README, materializer, and materializer tests are packaged Tested: codebase-graph setup refresh and graph-query count Not-tested: Hosted CI execution, Windows-specific process liveness semantics beyond local unit coverage. --- README.md | 3 +- src/codebase_graph/ingest/materializer.py | 58 ++++++++++++++++++----- tests/test_materializer.py | 23 ++++++++- 3 files changed, 69 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index c92b7e5..2dd8a96 100644 --- a/README.md +++ b/README.md @@ -182,6 +182,7 @@ Examples of emitted events include: - `mcp.stdio_parse_error` - `mcp.http_forbidden_origin` - `materializer.lock_exists` +- `materializer.stale_lock_removed` ## CLI graph workflow @@ -243,4 +244,4 @@ Report suspected vulnerabilities privately. See [SECURITY.md](SECURITY.md) for s - PATH or executable issues: run setup from the virtual environment that contains `codebase-graph`; the descriptor prefers that absolute executable path. - Direct smoke test: run `codebase-graph mcp serve --config .codebaseGraph/config.json` and send MCP `initialize`, `tools/list`, and `tools/call` JSON-RPC messages over stdio. - Unsupported files: binary, vendor, cache, virtualenv, build, dist, `.codebase_graph`, and `.codebaseGraph` paths are skipped. -- Lock/contention errors: stop other graph materialization or setup processes using the same `.codebaseGraph/_graph.ldb`. If no writer is running, remove the stale `.ldb.lock` file named in the error, then rerun setup. +- Lock/contention errors: stop other graph materialization or setup processes using the same `.codebaseGraph/_graph.ldb`. Stale locks with dead writer PIDs are removed automatically; if the error remains, inspect the `.ldb.lock` file before removing it manually. diff --git a/src/codebase_graph/ingest/materializer.py b/src/codebase_graph/ingest/materializer.py index 9c61757..ca3576c 100644 --- a/src/codebase_graph/ingest/materializer.py +++ b/src/codebase_graph/ingest/materializer.py @@ -527,19 +527,30 @@ def _temporary_sibling(path: Path, *, suffix: str) -> Path: def _acquire_materialization_lock(db_path: Path) -> tuple[int, Path]: lock_path = Path(f"{db_path}.lock") lock_path.parent.mkdir(parents=True, exist_ok=True) - try: - descriptor = os.open(lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY) - except FileExistsError as exc: - log_event( - "materializer.lock_exists", - level="WARNING", - db_path=db_path.as_posix(), - lock_path=lock_path.as_posix(), - ) - raise RuntimeError( - f"codebaseGraph materialization is already in progress for {db_path}. " - f"If no materializer is running, remove the stale lock file: {lock_path}" - ) from exc + while True: + try: + descriptor = os.open(lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY) + break + except FileExistsError as exc: + if _materialization_lock_is_stale(lock_path): + _unlink_if_exists(lock_path) + log_event( + "materializer.stale_lock_removed", + level="WARNING", + db_path=db_path.as_posix(), + lock_path=lock_path.as_posix(), + ) + continue + log_event( + "materializer.lock_exists", + level="WARNING", + db_path=db_path.as_posix(), + lock_path=lock_path.as_posix(), + ) + raise RuntimeError( + f"codebaseGraph materialization is already in progress for {db_path}. " + f"If no materializer is running, inspect the lock file before removing it: {lock_path}" + ) from exc payload = { "created_at": datetime.now(timezone.utc).isoformat(), "pid": os.getpid(), @@ -554,6 +565,27 @@ def _acquire_materialization_lock(db_path: Path) -> tuple[int, Path]: return descriptor, lock_path +def _materialization_lock_is_stale(lock_path: Path) -> bool: + try: + payload = json.loads(lock_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return False + pid = payload.get("pid") if isinstance(payload, dict) else None + if not isinstance(pid, int) or pid <= 0 or pid == os.getpid(): + return False + return not _process_is_running(pid) + + +def _process_is_running(pid: int) -> bool: + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + return True + + def _release_materialization_lock(descriptor: int, lock_path: Path) -> None: os.close(descriptor) _unlink_if_exists(lock_path) diff --git a/tests/test_materializer.py b/tests/test_materializer.py index 30f0a9b..376a9a3 100644 --- a/tests/test_materializer.py +++ b/tests/test_materializer.py @@ -1,5 +1,7 @@ from __future__ import annotations +import json +import os import shutil from pathlib import Path @@ -444,7 +446,7 @@ def test_ondisk_materialization_rejects_concurrent_writer_lock(tmp_path: Path) - db_path = tmp_path / "graph.lbug" manifest_path = tmp_path / "manifest.json" lock_path = Path(f"{db_path}.lock") - lock_path.write_text("{}\n", encoding="utf-8") + lock_path.write_text(json.dumps({"pid": os.getpid()}) + "\n", encoding="utf-8") with pytest.raises(RuntimeError, match="materialization is already in progress"): GraphMaterializer(source_root, db_path=db_path, manifest_path=manifest_path, include_fts=False).materialize( @@ -454,6 +456,25 @@ def test_ondisk_materialization_rejects_concurrent_writer_lock(tmp_path: Path) - assert lock_path.exists() +def test_ondisk_materialization_recovers_stale_writer_lock(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + pytest.importorskip("tree_sitter") + pytest.importorskip("tree_sitter_python") + pytest.importorskip("real_ladybug") + source_root = _copy_fixture(tmp_path) + db_path = tmp_path / "graph.lbug" + manifest_path = tmp_path / "manifest.json" + lock_path = Path(f"{db_path}.lock") + lock_path.write_text(json.dumps({"pid": 123456, "db_path": db_path.as_posix()}) + "\n", encoding="utf-8") + monkeypatch.setattr(materializer_module, "_process_is_running", lambda pid: False) + + result = GraphMaterializer(source_root, db_path=db_path, manifest_path=manifest_path, include_fts=False).materialize( + mode="full" + ) + + assert result.rebuilt == 4 + assert not lock_path.exists() + + def test_pending_rebuild_marker_forces_changed_mode_atomic_rebuild( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, From a704d3b61f7a1ec90a4632f8a71f4cee3b67628e Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Wed, 27 May 2026 16:54:03 +0930 Subject: [PATCH 32/53] fix: require MCP initialize before tool calls Rejects MCP tool listing and tool calls until the session has negotiated a protocol version through initialize. The HTTP transport now reuses one McpGraphServer per HTTP server so initialize state survives across requests instead of being discarded per POST. Constraint: Existing smoke clients initialize before tools but do not consistently send notifications/initialized. Rejected: Enforce notifications/initialized before every tool call | that would break current packaged smoke coverage and some clients while solving a narrower risk than pre-initialize tool access. Rejected: Keep a fresh MCP server per HTTP POST | it makes protocol session state impossible to honor over HTTP. Confidence: high Scope-risk: narrow Directive: If stricter MCP session semantics are added later, update packaged smoke clients and HTTP tests together. Tested: ./.venv/bin/python -m pytest Tested: ./.venv/bin/ruff check . Tested: ./.venv/bin/python scripts/check_release_gate.py Tested: ./.venv/bin/python -m build --no-isolation --outdir /private/tmp/codebasegraph-dist-mcp-session Tested: ./.venv/bin/python -m twine check /private/tmp/codebasegraph-dist-mcp-session/* Tested: ./.venv/bin/python -m pip check Tested: sdist tar listing confirms MCP protocol, HTTP transport, and MCP portability tests are packaged Tested: codebase-graph setup refresh and graph-query count Not-tested: Hosted CI execution; enforcement of notifications/initialized is intentionally deferred. --- src/codebase_graph/mcp/protocol.py | 2 ++ src/codebase_graph/mcp/transports/http.py | 4 +++- tests/test_mcp_portability.py | 17 +++++++++++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/codebase_graph/mcp/protocol.py b/src/codebase_graph/mcp/protocol.py index c466b50..e963755 100644 --- a/src/codebase_graph/mcp/protocol.py +++ b/src/codebase_graph/mcp/protocol.py @@ -50,6 +50,8 @@ def handle_json_rpc(self, message: dict[str, Any]) -> dict[str, Any] | None: return None if method.startswith("notifications/"): return None + if method in {"tools/list", "tools/call"} and self.session.protocol_version is None: + return rpc_error(request_id, -32002, "MCP session is not initialized") try: if method == "initialize": result = self._initialize(dict(message.get("params") or {})) diff --git a/src/codebase_graph/mcp/transports/http.py b/src/codebase_graph/mcp/transports/http.py index e06587c..f7729bb 100644 --- a/src/codebase_graph/mcp/transports/http.py +++ b/src/codebase_graph/mcp/transports/http.py @@ -20,6 +20,7 @@ class McpHttpServer(ThreadingHTTPServer): def __init__(self, server_address: tuple[str, int], handler: type[BaseHTTPRequestHandler]) -> None: super().__init__(server_address, handler) self.mcp_runtime: GraphRuntimeConfig + self.mcp_server: McpGraphServer self.endpoint_path: str self.auth_token: str | None @@ -52,6 +53,7 @@ def build_http_server( ) httpd = McpHttpServer((host, port), _McpHttpHandler) httpd.mcp_runtime = graph_runtime + httpd.mcp_server = McpGraphServer(graph_runtime) httpd.endpoint_path = endpoint_path httpd.auth_token = auth_token return httpd @@ -106,7 +108,7 @@ def do_POST(self) -> None: if not isinstance(message, dict): self._send_json(rpc_error(None, -32600, "JSON-RPC payload must be an object"), status=HTTPStatus.BAD_REQUEST) return - response = McpGraphServer(self.server.mcp_runtime).handle_json_rpc(message) + response = self.server.mcp_server.handle_json_rpc(message) if response is None: self.send_response(HTTPStatus.ACCEPTED) self.end_headers() diff --git a/tests/test_mcp_portability.py b/tests/test_mcp_portability.py index e3cf095..711dd17 100644 --- a/tests/test_mcp_portability.py +++ b/tests/test_mcp_portability.py @@ -49,6 +49,7 @@ def test_architecture_query_catalog_is_available_over_mcp_without_opening_graph( db_path.write_text("", encoding="utf-8") server = McpGraphServer(GraphRuntimeConfig(repo_root=tmp_path, db_path=db_path)) + server.handle_json_rpc({"jsonrpc": "2.0", "id": 0, "method": "initialize", "params": {}}) listed = server.handle_json_rpc({"jsonrpc": "2.0", "id": 1, "method": "tools/list", "params": {}}) all_queries = server.handle_json_rpc( { @@ -87,6 +88,22 @@ def test_architecture_query_catalog_is_available_over_mcp_without_opening_graph( assert invalid["result"]["structuredContent"]["error"]["type"] == "ValueError" +def test_mcp_rejects_tools_before_initialize(tmp_path: Path) -> None: + db_path = tmp_path / "graph.ldb" + db_path.write_text("", encoding="utf-8") + server = McpGraphServer(GraphRuntimeConfig(repo_root=tmp_path, db_path=db_path)) + + listed = server.handle_json_rpc({"jsonrpc": "2.0", "id": 1, "method": "tools/list", "params": {}}) + called = server.handle_json_rpc( + {"jsonrpc": "2.0", "id": 2, "method": "tools/call", "params": {"name": "graph_health", "arguments": {}}} + ) + + assert listed is not None + assert called is not None + assert listed["error"]["code"] == -32002 + assert called["error"]["code"] == -32002 + + def test_descriptor_prefers_current_environment_script(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: bin_dir = tmp_path / "venv" / "bin" bin_dir.mkdir(parents=True) From 64c061e23cbb7e94ed4b70945bbac701e2d7fc5f Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Wed, 27 May 2026 16:58:41 +0930 Subject: [PATCH 33/53] fix: bound runtime dependency versions Pins runtime dependencies to the tested compatible release families so production installs do not silently float onto unverified LadyBugDB or tree-sitter APIs. Packaging tests now assert those bounds are present in project metadata. Constraint: Verification in this workspace used real_ladybug 0.15.3, tree-sitter 0.25.2, tree-sitter-python 0.25.0, and tomli 2.4.1. Rejected: Leave runtime dependencies unbounded | upstream API drift could break production installs despite local tests passing today. Rejected: Pin exact versions | that would unnecessarily block patch-level compatible fixes. Confidence: medium Scope-risk: moderate Directive: Revisit these compatible-series bounds when intentionally validating newer LadyBugDB or tree-sitter minor versions. Tested: ./.venv/bin/python -m pytest Tested: ./.venv/bin/ruff check . Tested: ./.venv/bin/python scripts/check_release_gate.py Tested: ./.venv/bin/python -m build --no-isolation --outdir /private/tmp/codebasegraph-dist-dependency-bounds Tested: ./.venv/bin/python -m twine check /private/tmp/codebasegraph-dist-dependency-bounds/* Tested: ./.venv/bin/python -m pip check Tested: built PKG-INFO includes bounded Requires-Dist entries Tested: codebase-graph setup refresh and graph-query count Not-tested: Fresh dependency resolution from PyPI in a networked clean environment. --- pyproject.toml | 8 ++++---- tests/test_setup_workflow.py | 5 ++++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 258ac10..7928790 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,10 +10,10 @@ readme = "README.md" requires-python = ">=3.10" authors = [{ name = "Rabii Chaarani" }] dependencies = [ - "real_ladybug", - "tomli; python_version < '3.11'", - "tree-sitter", - "tree-sitter-python", + "real_ladybug>=0.15.3,<0.16", + "tomli>=2.0.1; python_version < '3.11'", + "tree-sitter>=0.25.2,<0.26", + "tree-sitter-python>=0.25.0,<0.26", ] classifiers = [ "Programming Language :: Python :: 3", diff --git a/tests/test_setup_workflow.py b/tests/test_setup_workflow.py index b2eda0b..1663e92 100644 --- a/tests/test_setup_workflow.py +++ b/tests/test_setup_workflow.py @@ -253,8 +253,11 @@ def test_setup_invalid_repo_root_exits_nonzero(tmp_path: Path) -> None: def test_packaging_requires_ladybug_and_namespaced_package_discovery() -> None: payload = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8")) + dependencies = "\n".join(payload["project"]["dependencies"]) - assert "real_ladybug" in payload["project"]["dependencies"] + assert "real_ladybug>=0.15.3,<0.16" in dependencies + assert "tree-sitter>=0.25.2,<0.26" in dependencies + assert "tree-sitter-python>=0.25.0,<0.26" in dependencies assert payload["project"]["scripts"]["codebase-graph"] == "codebase_graph.cli:main" assert payload["project"]["scripts"]["codebase-graph-mcp"] == "codebase_graph.mcp.server:main" assert payload["tool"]["setuptools"]["packages"]["find"]["include"] == ["codebase_graph*"] From ad74f8eaec547e9ffa2a83c4257a20546dbba0f2 Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Wed, 27 May 2026 17:05:02 +0930 Subject: [PATCH 34/53] fix: gate conda release version placeholders Makes the conda-forge recipe version release-specific instead of leaving a stale fixed version, extends the production release gate to catch that placeholder, and aligns recipe runtime dependency bounds with the tested package metadata ranges. Constraint: Conda-forge submission remains a post-PyPI or explicitly scoped release activity because the source distribution SHA256 is release-specific. Rejected: Keep version 0.1.0 in the staged recipe | it can become silently stale while SHA and license placeholders are replaced. Rejected: Leave conda runtime dependencies unbounded | that would diverge from the verified package metadata dependency policy. Confidence: high Scope-risk: narrow Directive: When conda-forge is in scope, replace version, sdist SHA256, and SPDX license together before running the gate with --require-conda. Tested: ./.venv/bin/python -m pytest Tested: ./.venv/bin/ruff check . Tested: ./.venv/bin/python scripts/check_release_gate.py Tested: ./.venv/bin/python scripts/check_release_gate.py --production --require-conda Tested: ./.venv/bin/python -m build --no-isolation --outdir /private/tmp/codebasegraph-dist-conda-gate Tested: ./.venv/bin/python -m twine check /private/tmp/codebasegraph-dist-conda-gate/* Tested: ./.venv/bin/python -m pip check Tested: sdist tar listing confirms conda recipe, release gate checker, and release workflow tests are packaged Tested: codebase-graph setup refresh and graph-query count Not-tested: Actual conda-forge staged-recipes CI; release-specific version/SHA/license values are still owner/release inputs. --- conda-forge/recipe/meta.yaml | 10 +++++----- scripts/check_release_gate.py | 2 +- tests/test_release_workflows.py | 11 +++++++++++ 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/conda-forge/recipe/meta.yaml b/conda-forge/recipe/meta.yaml index ed09224..ae3bf05 100644 --- a/conda-forge/recipe/meta.yaml +++ b/conda-forge/recipe/meta.yaml @@ -1,6 +1,6 @@ {% set name = "codebase-graph" %} {% set pypi_name = "codebase_graph" %} -{% set version = "0.1.0" %} +{% set version = "PUT_RELEASE_VERSION_HERE" %} {% set python_min = "3.10" %} package: @@ -28,10 +28,10 @@ requirements: - wheel run: - python >={{ python_min }} - - real-ladybug - - tomli # [py<311] - - tree-sitter - - tree-sitter-python + - real-ladybug >=0.15.3,<0.16 + - tomli >=2.0.1 # [py<311] + - tree-sitter >=0.25.2,<0.26 + - tree-sitter-python >=0.25.0,<0.26 test: imports: diff --git a/scripts/check_release_gate.py b/scripts/check_release_gate.py index 8ccb1ec..d39e2c1 100644 --- a/scripts/check_release_gate.py +++ b/scripts/check_release_gate.py @@ -209,7 +209,7 @@ def _check_external_confirmations(confirmations: set[str]) -> list[GateIssue]: def _check_conda_recipe() -> list[GateIssue]: recipe = (REPO_ROOT / "conda-forge/recipe/meta.yaml").read_text(encoding="utf-8") issues: list[GateIssue] = [] - for placeholder in ("PUT_RELEASE_SDIST_SHA256_HERE", "PUT_SPDX_LICENSE_ID_HERE"): + for placeholder in ("PUT_RELEASE_VERSION_HERE", "PUT_RELEASE_SDIST_SHA256_HERE", "PUT_SPDX_LICENSE_ID_HERE"): if placeholder in recipe: issues.append(GateIssue("FAIL", "conda-placeholder", f"conda recipe still contains {placeholder}.")) return issues diff --git a/tests/test_release_workflows.py b/tests/test_release_workflows.py index 7ebf360..d7a5abe 100644 --- a/tests/test_release_workflows.py +++ b/tests/test_release_workflows.py @@ -51,6 +51,15 @@ def test_release_workflow_enforces_production_gate_before_build() -> None: assert "build:\n name: build release distributions\n needs:\n - release-please\n - production-gate" in text +def test_conda_recipe_uses_bounded_runtime_dependencies() -> None: + text = Path("conda-forge/recipe/meta.yaml").read_text(encoding="utf-8") + + assert "real-ladybug >=0.15.3,<0.16" in text + assert "tomli >=2.0.1" in text + assert "tree-sitter >=0.25.2,<0.26" in text + assert "tree-sitter-python >=0.25.0,<0.26" in text + + def test_hosted_workflows_run_real_vulnerability_scans() -> None: for path in WORKFLOWS: text = path.read_text(encoding="utf-8") @@ -80,8 +89,10 @@ def test_local_release_gate_passes() -> None: def test_production_release_gate_reports_owner_controlled_blockers() -> None: issues = run_checks(production=True, require_conda=True, confirmations=set()) codes = {issue.code for issue in issues} + messages = {issue.message for issue in issues} assert "license-metadata-missing" in codes assert "license-file-missing" in codes assert "external-confirmation-missing" in codes assert "conda-placeholder" in codes + assert "conda recipe still contains PUT_RELEASE_VERSION_HERE." in messages From 7bb12db5bcf90889811eb5edeb5cbe73027a2d05 Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Wed, 27 May 2026 17:10:00 +0930 Subject: [PATCH 35/53] fix: publish project URLs in package metadata Adds PyPI project URLs for homepage, repository, issues, and changelog so production package consumers can find source, support, and release history directly from distribution metadata. Constraint: License metadata is still intentionally owner-selected and not inferred by this change. Rejected: Leave project links only in README | package indexes and installers surface Project-URL metadata directly. Confidence: high Scope-risk: narrow Directive: Keep project URLs aligned with repository ownership if the package is transferred. Tested: ./.venv/bin/python -m pytest Tested: ./.venv/bin/ruff check . Tested: ./.venv/bin/python scripts/check_release_gate.py Tested: ./.venv/bin/python -m build --no-isolation --outdir /private/tmp/codebasegraph-dist-project-urls Tested: ./.venv/bin/python -m twine check /private/tmp/codebasegraph-dist-project-urls/* Tested: ./.venv/bin/python -m pip check Tested: built PKG-INFO includes Homepage, Repository, Issues, and Changelog Project-URL entries Tested: codebase-graph setup refresh and graph-query count Not-tested: Rendering on PyPI production project page. --- pyproject.toml | 6 ++++++ tests/test_setup_workflow.py | 2 ++ 2 files changed, 8 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 7928790..2651740 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,12 @@ dev = ["pytest", "ruff"] codebase-graph = "codebase_graph.cli:main" codebase-graph-mcp = "codebase_graph.mcp.server:main" +[project.urls] +Homepage = "https://github.com/rabii-chaarani/codebaseGraph" +Repository = "https://github.com/rabii-chaarani/codebaseGraph" +Issues = "https://github.com/rabii-chaarani/codebaseGraph/issues" +Changelog = "https://github.com/rabii-chaarani/codebaseGraph/blob/main/CHANGELOG.md" + [tool.setuptools.packages.find] where = ["src"] include = ["codebase_graph*"] diff --git a/tests/test_setup_workflow.py b/tests/test_setup_workflow.py index 1663e92..7c956bb 100644 --- a/tests/test_setup_workflow.py +++ b/tests/test_setup_workflow.py @@ -260,6 +260,8 @@ def test_packaging_requires_ladybug_and_namespaced_package_discovery() -> None: assert "tree-sitter-python>=0.25.0,<0.26" in dependencies assert payload["project"]["scripts"]["codebase-graph"] == "codebase_graph.cli:main" assert payload["project"]["scripts"]["codebase-graph-mcp"] == "codebase_graph.mcp.server:main" + assert payload["project"]["urls"]["Repository"] == "https://github.com/rabii-chaarani/codebaseGraph" + assert payload["project"]["urls"]["Issues"] == "https://github.com/rabii-chaarani/codebaseGraph/issues" assert payload["tool"]["setuptools"]["packages"]["find"]["include"] == ["codebase_graph*"] From dda75c859600a77cc2314feeff31ca230520870e Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Wed, 27 May 2026 17:16:20 +0930 Subject: [PATCH 36/53] fix: report missing release gate files Keep the production release checker on the structured GateIssue path when required release files are absent. Missing release workflows and conda recipes now fail as actionable gate issues instead of escaping as FileNotFoundError tracebacks. Constraint: The production gate must still leave license, PyPI, hosted CI, vulnerability-reporting, and release-specific conda values owner-controlled. Rejected: Let missing files fail through Python exceptions | production operators need stable gate codes and messages. Confidence: high Scope-risk: narrow Directive: Keep release readiness failures structured so CI and operators can distinguish local gate defects from owner-controlled release blockers. Tested: ./.venv/bin/ruff check . Tested: ./.venv/bin/python -m pytest Tested: ./.venv/bin/python scripts/check_release_gate.py Tested: ./.venv/bin/python scripts/check_release_gate.py --production --require-conda Tested: ./.venv/bin/python -m build --no-isolation --outdir /private/tmp/codebasegraph-dist-release-gate-hardening Tested: ./.venv/bin/python -m twine check /private/tmp/codebasegraph-dist-release-gate-hardening/* Tested: ./.venv/bin/python -m pip check Tested: codebase-graph setup refresh and graph-query count Not-tested: External GitHub/PyPI production settings, which require owner account access. --- scripts/check_release_gate.py | 12 ++++++++++-- tests/test_release_workflows.py | 19 +++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/scripts/check_release_gate.py b/scripts/check_release_gate.py index d39e2c1..3bbb4e0 100644 --- a/scripts/check_release_gate.py +++ b/scripts/check_release_gate.py @@ -162,8 +162,12 @@ def _jobs_missing_timeout(text: str) -> list[str]: def _check_release_workflow_permissions() -> list[GateIssue]: - text = (REPO_ROOT / ".github/workflows/release.yml").read_text(encoding="utf-8") + workflow = REPO_ROOT / ".github/workflows/release.yml" issues: list[GateIssue] = [] + if not workflow.exists(): + return [GateIssue("FAIL", "workflow-missing", ".github/workflows/release.yml is required.")] + + text = workflow.read_text(encoding="utf-8") if ( "production-gate:" not in text or "python scripts/check_release_gate.py" not in text @@ -207,8 +211,12 @@ def _check_external_confirmations(confirmations: set[str]) -> list[GateIssue]: def _check_conda_recipe() -> list[GateIssue]: - recipe = (REPO_ROOT / "conda-forge/recipe/meta.yaml").read_text(encoding="utf-8") + recipe_path = REPO_ROOT / "conda-forge/recipe/meta.yaml" issues: list[GateIssue] = [] + if not recipe_path.exists(): + return [GateIssue("FAIL", "conda-recipe-missing", "conda-forge/recipe/meta.yaml is required.")] + + recipe = recipe_path.read_text(encoding="utf-8") for placeholder in ("PUT_RELEASE_VERSION_HERE", "PUT_RELEASE_SDIST_SHA256_HERE", "PUT_SPDX_LICENSE_ID_HERE"): if placeholder in recipe: issues.append(GateIssue("FAIL", "conda-placeholder", f"conda recipe still contains {placeholder}.")) diff --git a/tests/test_release_workflows.py b/tests/test_release_workflows.py index d7a5abe..089e918 100644 --- a/tests/test_release_workflows.py +++ b/tests/test_release_workflows.py @@ -3,6 +3,7 @@ import re from pathlib import Path +from scripts import check_release_gate from scripts.check_release_gate import _jobs_missing_timeout, _workflow_action_pin_issues, run_checks @@ -96,3 +97,21 @@ def test_production_release_gate_reports_owner_controlled_blockers() -> None: assert "external-confirmation-missing" in codes assert "conda-placeholder" in codes assert "conda recipe still contains PUT_RELEASE_VERSION_HERE." in messages + + +def test_release_gate_reports_missing_release_workflow(monkeypatch, tmp_path) -> None: + monkeypatch.setattr(check_release_gate, "REPO_ROOT", tmp_path) + + issues = check_release_gate._check_release_workflow_permissions() + + assert [issue.code for issue in issues] == ["workflow-missing"] + assert ".github/workflows/release.yml is required." in issues[0].message + + +def test_release_gate_reports_missing_conda_recipe(monkeypatch, tmp_path) -> None: + monkeypatch.setattr(check_release_gate, "REPO_ROOT", tmp_path) + + issues = check_release_gate._check_conda_recipe() + + assert [issue.code for issue in issues] == ["conda-recipe-missing"] + assert "conda-forge/recipe/meta.yaml is required." in issues[0].message From 9dac57e75777e85c9afa29041c89c8e4150af4b4 Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Wed, 27 May 2026 17:39:43 +0930 Subject: [PATCH 37/53] fix: harden MCP runtime and release smoke gates Tightens production boundaries around runtime identity, HTTP MCP sessions, raw graph queries, and release smoke coverage. Setup configs now validate their rooted path invariants, explicit configs drive the reported runtime repo identity, HTTP requests carry session ids instead of sharing a listener-wide MCP session, and packaged smoke tests exercise mcp install --verify through an isolated client config path. Constraint: License selection, PyPI/GitHub confirmations, hosted CI status, private vulnerability reporting, and release-specific conda metadata remain owner-controlled production gates. Rejected: Keep HTTP compatibility through a listener-global session | one initialize request must not unlock tool calls for unrelated HTTP clients. Rejected: Isolate packaged installer smoke by changing HOME | that perturbs LadyBugDB extension resolution and hides the actual installer contract. Confidence: high Scope-risk: moderate Directive: Keep setup config paths rooted under the configured repo .codebaseGraph directory unless a future explicit unsafe override includes dedicated tests and docs. Directive: Keep HTTP MCP clients carrying Mcp-Session-Id after initialize; do not reintroduce listener-global session unlocks. Tested: ./.venv/bin/ruff check . Tested: ./.venv/bin/python -m pytest Tested: ./.venv/bin/python scripts/check_release_gate.py Tested: ./.venv/bin/python scripts/check_release_gate.py --production --require-conda Tested: ./.venv/bin/python -m build --no-isolation --outdir /private/tmp/codebasegraph-dist-prod-hardening Tested: ./.venv/bin/python -m twine check /private/tmp/codebasegraph-dist-prod-hardening/* Tested: ./.venv/bin/python -m pip check Tested: ./.venv/bin/python scripts/smoke_built_wheel.py ./.venv/bin/codebase-graph Tested: codebase-graph setup refresh and graph-query count Not-tested: Hosted GitHub Actions matrix, PyPI Trusted Publisher settings, private vulnerability reporting settings, and conda-forge staged-recipes CI. --- README.md | 3 + SECURITY.md | 1 + scripts/smoke_built_wheel.py | 27 +++++++++ src/codebase_graph/cli/__init__.py | 2 + src/codebase_graph/mcp/runtime.py | 1 + src/codebase_graph/mcp/tools.py | 5 +- src/codebase_graph/mcp/transports/http.py | 31 ++++++++-- src/codebase_graph/setup/state.py | 22 +++++++- tests/test_mcp_installer.py | 27 +++++++++ tests/test_mcp_portability.py | 69 +++++++++++++++++++++-- tests/test_release_workflows.py | 18 +++++- tests/test_search.py | 18 ++++++ tests/test_setup_workflow.py | 31 ++++++++++ 13 files changed, 244 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 2dd8a96..50a9a99 100644 --- a/README.md +++ b/README.md @@ -155,6 +155,9 @@ codebase-graph mcp http --config .codebaseGraph/config.json --host 0.0.0.0 --all Clients must send `Authorization: Bearer `. The token gate does not add TLS, rate limiting, authorization scopes, or a multi-user session model; put remote HTTP behind a trusted network boundary and TLS-terminating proxy. +HTTP clients must start with JSON-RPC `initialize`, then send the returned `Mcp-Session-Id` response header on later +requests. Requests without a known session id are rejected before tool dispatch. + Available MCP tools: - `graph_health` diff --git a/SECURITY.md b/SECURITY.md index 17a3334..45168f1 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -25,6 +25,7 @@ The production security boundary is local-first: - The stdio MCP transport is intended for local MCP clients. - The HTTP MCP transport binds to localhost by default. - `--allow-remote` requires a bearer token. It does not add TLS, rate limiting, authorization scopes, or a multi-user session model. +- HTTP tool calls require an initialized `Mcp-Session-Id`; one client's initialize request must not unlock tools for another client. - `graph_query` is intended to remain read-only. Do not relax query restrictions without a parser-level read-only proof or an explicit safe-procedure allowlist. Dependency vulnerability scanning runs in hosted CI and release workflows. Local setup commands must not call external advisory services implicitly. diff --git a/scripts/smoke_built_wheel.py b/scripts/smoke_built_wheel.py index 27a60d8..9825ca2 100644 --- a/scripts/smoke_built_wheel.py +++ b/scripts/smoke_built_wheel.py @@ -51,6 +51,7 @@ def main(argv: list[str]) -> int: if not search.get("results"): raise AssertionError(f"graph-search returned no results: {search}") + _install_verify_smoke(executable, config_path, Path(tmp_dir) / "mcp.json") _mcp_smoke([executable.as_posix(), "mcp", "serve", "--config", config_path.as_posix()]) return 0 @@ -59,6 +60,32 @@ def _run(command: list[str]) -> subprocess.CompletedProcess[str]: return subprocess.run(command, capture_output=True, text=True, check=True) +def _install_verify_smoke(executable: Path, config_path: Path, client_config_path: Path) -> None: + verify = json.loads( + _run( + [ + executable.as_posix(), + "mcp", + "install", + "--client", + "generic", + "--config-path", + config_path.as_posix(), + "--client-config-path", + client_config_path.as_posix(), + "--verify", + "--json", + ] + ).stdout + ) + verification = verify.get("verification") or {} + stdio = verification.get("stdio") or {} + checks = stdio.get("checks") or {} + required_checks = ("initialize", "tools_list", "graph_health", "graph_search", "tool_error_result") + if verification.get("ok") is not True or not all(checks.get(check) is True for check in required_checks): + raise AssertionError(f"mcp install --verify failed readiness smoke: {verify}") + + def _sample_repo(repo_root: Path) -> Path: package = repo_root / "sample_project" package.mkdir(parents=True) diff --git a/src/codebase_graph/cli/__init__.py b/src/codebase_graph/cli/__init__.py index 3a52241..2b8af75 100644 --- a/src/codebase_graph/cli/__init__.py +++ b/src/codebase_graph/cli/__init__.py @@ -96,6 +96,7 @@ def main(argv: Sequence[str] | None = None) -> int: install_parser.add_argument("--scope", choices=("local", "user", "project"), default="local") install_parser.add_argument("--name", default=None, help="MCP server name; defaults to codebase_graph-") install_parser.add_argument("--config-path", default=None, help="Path to .codebaseGraph/config.json") + install_parser.add_argument("--client-config-path", default=None, help="Override the target MCP client config path") install_parser.add_argument("--repo-root", default=".", help="Repository root used to find .codebaseGraph/config.json") install_parser.add_argument("--dry-run", action="store_true", help="Show the install action without writing or invoking CLIs") install_parser.add_argument("--verify", action="store_true", help="Run direct MCP smoke checks after installation") @@ -242,6 +243,7 @@ def main(argv: Sequence[str] | None = None) -> int: scope=args.scope, setup_config_path=setup_config_path, server_name=args.name, + client_config_path=args.client_config_path, dry_run=args.dry_run, verify=args.verify, ) diff --git a/src/codebase_graph/mcp/runtime.py b/src/codebase_graph/mcp/runtime.py index 760f3d2..70f44ce 100644 --- a/src/codebase_graph/mcp/runtime.py +++ b/src/codebase_graph/mcp/runtime.py @@ -28,6 +28,7 @@ def runtime_config( payload: dict[str, Any] = {} if config.exists(): payload = load_setup_config(config) + root = Path(str(payload["repo_root"])).expanduser().resolve() elif db_path is None: raise FileNotFoundError(f"codebaseGraph setup config is missing: {config}") resolved_db = Path(db_path or payload["database_path"]).expanduser().resolve() diff --git a/src/codebase_graph/mcp/tools.py b/src/codebase_graph/mcp/tools.py index ba74fbe..796b533 100644 --- a/src/codebase_graph/mcp/tools.py +++ b/src/codebase_graph/mcp/tools.py @@ -13,7 +13,10 @@ from .runtime import GraphRuntimeConfig, open_graph_store READ_ONLY_DENY_RE = re.compile( - r"\b(CALL|CREATE|DELETE|SET|MERGE|DROP|COPY|INSERT|LOAD|INSTALL|DETACH|REMOVE|ALTER|RENAME)\b", + r"\b(" + r"ALTER|ATTACH|CALL|COPY|CREATE|DELETE|DETACH|DROP|EXPORT|IMPORT|INSERT|INSTALL|LOAD|MERGE|REMOVE|RENAME|SET|" + r"TRUNCATE|UPDATE|USE" + r")\b", re.IGNORECASE, ) MAX_GRAPH_QUERY_LIMIT = 1000 diff --git a/src/codebase_graph/mcp/transports/http.py b/src/codebase_graph/mcp/transports/http.py index f7729bb..af1db03 100644 --- a/src/codebase_graph/mcp/transports/http.py +++ b/src/codebase_graph/mcp/transports/http.py @@ -20,7 +20,7 @@ class McpHttpServer(ThreadingHTTPServer): def __init__(self, server_address: tuple[str, int], handler: type[BaseHTTPRequestHandler]) -> None: super().__init__(server_address, handler) self.mcp_runtime: GraphRuntimeConfig - self.mcp_server: McpGraphServer + self.mcp_sessions: dict[str, McpGraphServer] self.endpoint_path: str self.auth_token: str | None @@ -53,7 +53,7 @@ def build_http_server( ) httpd = McpHttpServer((host, port), _McpHttpHandler) httpd.mcp_runtime = graph_runtime - httpd.mcp_server = McpGraphServer(graph_runtime) + httpd.mcp_sessions = {} httpd.endpoint_path = endpoint_path httpd.auth_token = auth_token return httpd @@ -108,12 +108,16 @@ def do_POST(self) -> None: if not isinstance(message, dict): self._send_json(rpc_error(None, -32600, "JSON-RPC payload must be an object"), status=HTTPStatus.BAD_REQUEST) return - response = self.server.mcp_server.handle_json_rpc(message) + session_id, server = self._resolve_session(message) + if server is None: + return + response = server.handle_json_rpc(message) if response is None: self.send_response(HTTPStatus.ACCEPTED) self.end_headers() return - self._send_json(response) + headers = {"Mcp-Session-Id": session_id} if str(message.get("method", "")) == "initialize" else None + self._send_json(response, headers=headers) def do_GET(self) -> None: if not self._request_path_matches() or not self._valid_origin() or not self._valid_auth(): @@ -125,6 +129,25 @@ def do_GET(self) -> None: def log_message(self, format: str, *args: Any) -> None: return + def _resolve_session(self, message: dict[str, Any]) -> tuple[str, McpGraphServer] | tuple[None, None]: + method = str(message.get("method", "")) + request_id = message.get("id") + session_id = self.headers.get("Mcp-Session-Id") + if method == "initialize": + if session_id and session_id in self.server.mcp_sessions: + return session_id, self.server.mcp_sessions[session_id] + session_id = secrets.token_urlsafe(32) + server = McpGraphServer(self.server.mcp_runtime) + self.server.mcp_sessions[session_id] = server + return session_id, server + if not session_id or session_id not in self.server.mcp_sessions: + self._send_json( + rpc_error(request_id, -32002, "MCP session is not initialized"), + status=HTTPStatus.BAD_REQUEST, + ) + return None, None + return session_id, self.server.mcp_sessions[session_id] + def _request_path_matches(self) -> bool: if urlparse(self.path).path == self.server.endpoint_path: return True diff --git a/src/codebase_graph/setup/state.py b/src/codebase_graph/setup/state.py index 7aecd7d..d1baa08 100644 --- a/src/codebase_graph/setup/state.py +++ b/src/codebase_graph/setup/state.py @@ -87,8 +87,28 @@ def _read_json_if_exists(path: Path) -> dict[str, Any] | None: def _validate_setup_config(payload: dict[str, Any], path: Path) -> None: - required = ("repo_root", "repo_name", "database_path", "manifest_path") + required = ("repo_root", "repo_name", "state_dir", "database_path", "manifest_path") missing = [key for key in required if not payload.get(key)] if missing: joined = ", ".join(missing) raise ValueError(f"Invalid codebaseGraph setup config at {path}: missing {joined}") + + repo_root = Path(str(payload["repo_root"])).expanduser().resolve() + repo_name = str(payload["repo_name"]) + state_dir = Path(str(payload["state_dir"])).expanduser().resolve() + database_path = Path(str(payload["database_path"])).expanduser().resolve() + manifest_path = Path(str(payload["manifest_path"])).expanduser().resolve() + expected_state_dir = repo_root / DEFAULT_STATE_DIR + expected_database_path = state_dir / f"{repo_name}_graph.ldb" + expected_manifest_path = state_dir / MANIFEST_NAME + + if DEFAULT_STATE_DIR in repo_root.parts: + raise ValueError(f"Invalid codebaseGraph setup config at {path}: repo_root may not be inside {DEFAULT_STATE_DIR}") + if state_dir != expected_state_dir: + raise ValueError(f"Invalid codebaseGraph setup config at {path}: state_dir must be {expected_state_dir}") + if path.parent.resolve() != state_dir: + raise ValueError(f"Invalid codebaseGraph setup config at {path}: config must live under {state_dir}") + if database_path != expected_database_path: + raise ValueError(f"Invalid codebaseGraph setup config at {path}: database_path must be {expected_database_path}") + if manifest_path != expected_manifest_path: + raise ValueError(f"Invalid codebaseGraph setup config at {path}: manifest_path must be {expected_manifest_path}") diff --git a/tests/test_mcp_installer.py b/tests/test_mcp_installer.py index ca2f3f1..b238607 100644 --- a/tests/test_mcp_installer.py +++ b/tests/test_mcp_installer.py @@ -243,6 +243,33 @@ def test_mcp_install_cli_dry_run_json( assert output["server_name"] == default_server_name("fresh_repo") +def test_mcp_install_cli_accepts_client_config_path( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + config_path = _write_setup_config(tmp_path / "fresh_repo") + client_config_path = tmp_path / "client" / "mcp.json" + + exit_code = cli_main( + [ + "mcp", + "install", + "--client", + "generic", + "--config-path", + config_path.as_posix(), + "--client-config-path", + client_config_path.as_posix(), + "--json", + ] + ) + output = json.loads(capsys.readouterr().out) + + assert exit_code == 0 + assert output["path"] == client_config_path.as_posix() + assert client_config_path.exists() + + def _write_setup_config(repo_root: Path) -> Path: repo_root.mkdir(parents=True) paths = derive_setup_paths(repo_root) diff --git a/tests/test_mcp_portability.py b/tests/test_mcp_portability.py index 711dd17..ec44dcc 100644 --- a/tests/test_mcp_portability.py +++ b/tests/test_mcp_portability.py @@ -280,9 +280,17 @@ def test_http_mcp_transport_handles_initialize_list_and_call(tmp_path: Path) -> thread.start() host, port = httpd.server_address try: - initialize = _http_rpc(host, port, "initialize", {"protocolVersion": "2025-11-25"}) - listed = _http_rpc(host, port, "tools/list", {}) - health = _http_rpc(host, port, "tools/call", {"name": "graph_health", "arguments": {}}) + initialize, session_id = _http_rpc_with_session(host, port, "initialize", {"protocolVersion": "2025-11-25"}) + with pytest.raises(urllib.error.HTTPError) as missing_session: + _http_rpc(host, port, "tools/list", {}) + listed = _http_rpc(host, port, "tools/list", {}, session_id=session_id) + health = _http_rpc( + host, + port, + "tools/call", + {"name": "graph_health", "arguments": {}}, + session_id=session_id, + ) with pytest.raises(urllib.error.HTTPError) as exc_info: _http_rpc(host, port, "ping", {}, protocol_version="1900-01-01") finally: @@ -291,6 +299,7 @@ def test_http_mcp_transport_handles_initialize_list_and_call(tmp_path: Path) -> thread.join(timeout=10) assert initialize["result"]["protocolVersion"] == "2025-11-25" + assert missing_session.value.code == 400 assert any(tool["name"] == "graph_context" for tool in listed["result"]["tools"]) assert health["result"]["structuredContent"]["ok"] is True assert exc_info.value.code == 400 @@ -388,7 +397,57 @@ def _http_rpc( protocol_version: str = "2025-11-25", auth_token: str | None = None, origin: str | None = None, + session_id: str | None = None, ) -> dict[str, Any]: + return _http_rpc_with_headers( + host, + port, + method, + params, + protocol_version=protocol_version, + auth_token=auth_token, + origin=origin, + session_id=session_id, + )[0] + + +def _http_rpc_with_session( + host: str, + port: int, + method: str, + params: dict[str, Any], + *, + protocol_version: str = "2025-11-25", + auth_token: str | None = None, + origin: str | None = None, + session_id: str | None = None, +) -> tuple[dict[str, Any], str]: + payload, headers = _http_rpc_with_headers( + host, + port, + method, + params, + protocol_version=protocol_version, + auth_token=auth_token, + origin=origin, + session_id=session_id, + ) + resolved_session_id = headers.get("Mcp-Session-Id") + assert resolved_session_id + return payload, resolved_session_id + + +def _http_rpc_with_headers( + host: str, + port: int, + method: str, + params: dict[str, Any], + *, + protocol_version: str = "2025-11-25", + auth_token: str | None = None, + origin: str | None = None, + session_id: str | None = None, +) -> tuple[dict[str, Any], Any]: payload = json.dumps({"jsonrpc": "2.0", "id": 1, "method": method, "params": params}).encode("utf-8") headers = { "Accept": "application/json, text/event-stream", @@ -398,6 +457,8 @@ def _http_rpc( } if auth_token is not None: headers["Authorization"] = f"Bearer {auth_token}" + if session_id is not None: + headers["Mcp-Session-Id"] = session_id request = urllib.request.Request( f"http://{host}:{port}/mcp", data=payload, @@ -405,7 +466,7 @@ def _http_rpc( method="POST", ) with urllib.request.urlopen(request, timeout=10) as response: - return json.loads(response.read().decode("utf-8")) + return json.loads(response.read().decode("utf-8")), response.headers def _fresh_repo(tmp_path: Path) -> Path: diff --git a/tests/test_release_workflows.py b/tests/test_release_workflows.py index 089e918..083c3fb 100644 --- a/tests/test_release_workflows.py +++ b/tests/test_release_workflows.py @@ -4,7 +4,12 @@ from pathlib import Path from scripts import check_release_gate -from scripts.check_release_gate import _jobs_missing_timeout, _workflow_action_pin_issues, run_checks +from scripts.check_release_gate import ( + PYPI_CONFIRMATION_FLAGS, + _jobs_missing_timeout, + _workflow_action_pin_issues, + run_checks, +) WORKFLOWS = ( @@ -76,6 +81,17 @@ def test_security_policy_exists() -> None: assert "--allow-remote" in text +def test_release_docs_list_production_confirmation_flags() -> None: + text = Path("docs/release.md").read_text(encoding="utf-8") + + for flag in PYPI_CONFIRMATION_FLAGS: + env_var = f"CODEBASE_GRAPH_CONFIRM_{flag.upper().replace('-', '_')}" + assert env_var in text + assert f"--confirm {flag}" in text + assert "CODEBASE_GRAPH_REQUIRE_CONDA" in text + assert "--require-conda" in text + + def test_workflow_jobs_have_timeouts() -> None: for path in WORKFLOWS: missing = _jobs_missing_timeout(path.read_text(encoding="utf-8")) diff --git a/tests/test_search.py b/tests/test_search.py index ebf4641..efa5787 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -635,6 +635,24 @@ def test_graph_query_rejects_procedure_calls() -> None: _query_payload(store, {"statement": "CALL CREATE_FTS_INDEX('File', 'label')"}) +@pytest.mark.parametrize( + ("statement", "keyword"), + [ + ("EXPORT DATABASE '/tmp/graph-export'", "EXPORT"), + ("IMPORT DATABASE '/tmp/graph-export'", "IMPORT"), + ("ATTACH '/tmp/other.ldb' AS other", "ATTACH"), + ("USE other", "USE"), + ("TRUNCATE TABLE File", "TRUNCATE"), + ("UPDATE File SET label = 'x'", "UPDATE"), + ], +) +def test_graph_query_rejects_database_administration_statements(statement: str, keyword: str) -> None: + store = _RecordingStore([[1]]) + + with pytest.raises(ValueError, match=f"blocked keyword: {keyword}"): + _query_payload(store, {"statement": statement}) + + def _require_graph_runtime() -> None: pytest.importorskip("tree_sitter") pytest.importorskip("tree_sitter_python") diff --git a/tests/test_setup_workflow.py b/tests/test_setup_workflow.py index 7c956bb..ede7040 100644 --- a/tests/test_setup_workflow.py +++ b/tests/test_setup_workflow.py @@ -13,10 +13,12 @@ from codebase_graph.cli import main as cli_main from codebase_graph.db import LadybugUnavailableError +from codebase_graph.mcp.runtime import runtime_config from codebase_graph.mcp.server import McpGraphServer, handle_tool_call from codebase_graph.setup import SetupError, SetupOptions, run_setup from codebase_graph.setup.instructions import END_MARKER, START_MARKER from codebase_graph.setup.mcp_config import configure_mcp_client, server_entry +from codebase_graph.setup.state import build_setup_config, derive_setup_paths, load_setup_config, write_setup_config def test_setup_cli_creates_state_db_mcp_config_instructions_and_searchable_docs( @@ -251,6 +253,35 @@ def test_setup_invalid_repo_root_exits_nonzero(tmp_path: Path) -> None: assert exc_info.value.code == 2 +def test_runtime_config_uses_repo_root_from_setup_config(tmp_path: Path) -> None: + repo_root = _fresh_repo(tmp_path) + paths = derive_setup_paths(repo_root) + payload = build_setup_config(paths, mcp_command=["codebase-graph", "mcp", "serve", "--config", paths.config_path.as_posix()]) + write_setup_config(paths.config_path, payload) + paths.db_path.write_text("", encoding="utf-8") + paths.manifest_path.write_text("{}", encoding="utf-8") + other_root = tmp_path / "other_repo" + other_root.mkdir() + + runtime = runtime_config(repo_root=other_root, config_path=paths.config_path, db_path=None, manifest_path=None) + + assert runtime.repo_root == repo_root.resolve() + assert runtime.db_path == paths.db_path + assert runtime.manifest_path == paths.manifest_path + + +def test_setup_config_rejects_database_path_outside_state_dir(tmp_path: Path) -> None: + repo_root = _fresh_repo(tmp_path) + paths = derive_setup_paths(repo_root) + payload = build_setup_config(paths, mcp_command=["codebase-graph", "mcp", "serve", "--config", paths.config_path.as_posix()]) + payload["database_path"] = (tmp_path / "other.ldb").as_posix() + paths.config_path.parent.mkdir(parents=True) + paths.config_path.write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(ValueError, match="database_path must be"): + load_setup_config(paths.config_path) + + def test_packaging_requires_ladybug_and_namespaced_package_discovery() -> None: payload = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8")) dependencies = "\n".join(payload["project"]["dependencies"]) From eb083823f1da39cf654db542172d6faab2c7bbd3 Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Thu, 28 May 2026 09:20:02 +0930 Subject: [PATCH 38/53] feat: add MIT License --- LICENSE | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..82f2e19 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Rabii Chaarani + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From 21c51f4224e3b51ad6794624e6433a775053aa37 Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Thu, 28 May 2026 09:25:21 +0930 Subject: [PATCH 39/53] fix: update PyPI URL --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f4259d8..0418d73 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -211,7 +211,7 @@ jobs: timeout-minutes: 10 environment: name: pypi - url: https://pypi.org/p/codebase-graph + url: https://pypi.org/p/cbasegraph permissions: id-token: write From e7c8e17dc798f6cc189358f56ad355fea8b64362 Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Thu, 28 May 2026 09:43:04 +0930 Subject: [PATCH 40/53] fix: publish MIT license metadata Record the selected MIT license in package metadata and the conda recipe. The release-gate tests now assert that license metadata is resolved while leaving only owner confirmations and release-specific conda version/hash placeholders outside this task. Constraint: Existing tracked LICENSE declares MIT, so package and conda metadata now use that selected SPDX identifier. Rejected: Leave conda license placeholder unresolved | the SPDX license is known even though version and sdist SHA are release-specific. Confidence: high Scope-risk: narrow Directive: Do not change the SPDX license without updating LICENSE, pyproject metadata, conda recipe metadata, and release-gate tests together. Tested: ./.venv/bin/ruff check ., ./.venv/bin/python -m pytest -q, ./.venv/bin/python scripts/check_release_gate.py --production --confirm trusted-publisher --confirm pypi-environment --confirm hosted-ci-green --confirm private-vulnerability-reporting, ./.venv/bin/python -m build --no-isolation --outdir /private/tmp/codebasegraph-dist-license.K6QKsh, ./.venv/bin/python -m twine check /private/tmp/codebasegraph-dist-license.K6QKsh/* Not-tested: PyPI production publishing and conda-forge staged-recipes CI. --- conda-forge/recipe/meta.yaml | 4 ++-- pyproject.toml | 4 +++- tests/test_release_workflows.py | 9 +++++++-- tests/test_setup_workflow.py | 3 +++ 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/conda-forge/recipe/meta.yaml b/conda-forge/recipe/meta.yaml index ae3bf05..a91644f 100644 --- a/conda-forge/recipe/meta.yaml +++ b/conda-forge/recipe/meta.yaml @@ -23,7 +23,7 @@ requirements: host: - python {{ python_min }} - pip - - setuptools >=68 + - setuptools >=77 - setuptools-scm >=8 - wheel run: @@ -45,7 +45,7 @@ test: about: home: https://github.com/rabii-chaarani/codebaseGraph summary: Generic codebase knowledge graph engine for Python projects. - license: PUT_SPDX_LICENSE_ID_HERE + license: MIT extra: recipe-maintainers: diff --git a/pyproject.toml b/pyproject.toml index 2651740..784dfcf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=68", "setuptools-scm>=8", "wheel"] +requires = ["setuptools>=77", "setuptools-scm>=8", "wheel"] build-backend = "setuptools.build_meta" [project] @@ -9,6 +9,8 @@ description = "Generic codebase knowledge graph engine for Python projects." readme = "README.md" requires-python = ">=3.10" authors = [{ name = "Rabii Chaarani" }] +license = "MIT" +license-files = ["LICENSE"] dependencies = [ "real_ladybug>=0.15.3,<0.16", "tomli>=2.0.1; python_version < '3.11'", diff --git a/tests/test_release_workflows.py b/tests/test_release_workflows.py index 083c3fb..aa284ab 100644 --- a/tests/test_release_workflows.py +++ b/tests/test_release_workflows.py @@ -60,10 +60,13 @@ def test_release_workflow_enforces_production_gate_before_build() -> None: def test_conda_recipe_uses_bounded_runtime_dependencies() -> None: text = Path("conda-forge/recipe/meta.yaml").read_text(encoding="utf-8") + assert "setuptools >=77" in text assert "real-ladybug >=0.15.3,<0.16" in text assert "tomli >=2.0.1" in text assert "tree-sitter >=0.25.2,<0.26" in text assert "tree-sitter-python >=0.25.0,<0.26" in text + assert "license: MIT" in text + assert "PUT_SPDX_LICENSE_ID_HERE" not in text def test_hosted_workflows_run_real_vulnerability_scans() -> None: @@ -108,11 +111,13 @@ def test_production_release_gate_reports_owner_controlled_blockers() -> None: codes = {issue.code for issue in issues} messages = {issue.message for issue in issues} - assert "license-metadata-missing" in codes - assert "license-file-missing" in codes + assert "license-metadata-missing" not in codes + assert "license-file-missing" not in codes assert "external-confirmation-missing" in codes assert "conda-placeholder" in codes assert "conda recipe still contains PUT_RELEASE_VERSION_HERE." in messages + assert "conda recipe still contains PUT_RELEASE_SDIST_SHA256_HERE." in messages + assert "conda recipe still contains PUT_SPDX_LICENSE_ID_HERE." not in messages def test_release_gate_reports_missing_release_workflow(monkeypatch, tmp_path) -> None: diff --git a/tests/test_setup_workflow.py b/tests/test_setup_workflow.py index ede7040..3fcda65 100644 --- a/tests/test_setup_workflow.py +++ b/tests/test_setup_workflow.py @@ -289,6 +289,9 @@ def test_packaging_requires_ladybug_and_namespaced_package_discovery() -> None: assert "real_ladybug>=0.15.3,<0.16" in dependencies assert "tree-sitter>=0.25.2,<0.26" in dependencies assert "tree-sitter-python>=0.25.0,<0.26" in dependencies + assert "setuptools>=77" in payload["build-system"]["requires"] + assert payload["project"]["license"] == "MIT" + assert payload["project"]["license-files"] == ["LICENSE"] assert payload["project"]["scripts"]["codebase-graph"] == "codebase_graph.cli:main" assert payload["project"]["scripts"]["codebase-graph-mcp"] == "codebase_graph.mcp.server:main" assert payload["project"]["urls"]["Repository"] == "https://github.com/rabii-chaarani/codebaseGraph" From 0def62160a2890ce1db33074226de29250563171 Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Thu, 28 May 2026 10:00:06 +0930 Subject: [PATCH 41/53] fix: align CI audits with cbasegraph metadata The failed Actions run exposed two release-surface mismatches: strict pip-audit was auditing an editable or unpublished local distribution, and package metadata still used the old codebase-graph PyPI name. Align the distribution name with the published cbasegraph project and audit dependency declarations with project-path pip-audit in CI and release workflows. Constraint: PyPI project name is cbasegraph while CLI entry points remain codebase-graph and codebase-graph-mcp Rejected: Keep environment-level pip-audit on the local package | dev and pre-publish versions are not necessarily present on PyPI Confidence: high Scope-risk: moderate Directive: Do not reintroduce strict environment auditing of the local distribution before publish; audit project dependencies and keep pip check for installed wheel consistency Tested: .venv/bin/python -m pytest -q Tested: .venv/bin/ruff check . Tested: .venv/bin/python -m pip_audit --strict . --progress-spinner off --cache-dir /private/tmp/codebase-graph-audit-pip-cache Tested: .venv/bin/python -m build --outdir /private/tmp/codebase-graph-dist Tested: .venv/bin/python -m twine check /private/tmp/codebase-graph-dist/* Not-tested: Hosted GitHub Actions rerun after push --- .github/workflows/ci.yml | 4 ++-- .github/workflows/release.yml | 2 +- README.md | 2 +- conda-forge/recipe/meta.yaml | 4 ++-- docs/release.md | 2 +- pyproject.toml | 2 +- tests/test_release_workflows.py | 21 +++++++++++++++++++++ 7 files changed, 29 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d5b37eb..190844c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -97,14 +97,14 @@ jobs: - name: Install package and supply-chain tools run: | python -m pip install --upgrade pip - python -m pip install -e ".[dev]" + python -m pip install ".[dev]" python -m pip install cyclonedx-bom pip-audit - name: Check installed dependency consistency run: python -m pip check - name: Run vulnerability advisory scan - run: python -m pip_audit --strict --skip-editable --progress-spinner off --cache-dir .pip-audit-cache + run: python -m pip_audit --strict . --progress-spinner off --cache-dir .pip-audit-cache - name: Generate CycloneDX SBOM run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0418d73..cab4c86 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -164,7 +164,7 @@ jobs: /tmp/codebase-graph-wheel/bin/codebase-graph-mcp --help /tmp/codebase-graph-wheel/bin/python scripts/smoke_built_wheel.py /tmp/codebase-graph-wheel/bin/codebase-graph /tmp/codebase-graph-wheel/bin/python -m pip check - /tmp/codebase-graph-wheel/bin/python -m pip_audit --strict --skip-editable --progress-spinner off --cache-dir /tmp/pip-audit-cache + /tmp/codebase-graph-wheel/bin/python -m pip_audit --strict . --progress-spinner off --cache-dir /tmp/pip-audit-cache /tmp/codebase-graph-wheel/bin/cyclonedx-py environment /tmp/codebase-graph-wheel/bin/python \ --pyproject pyproject.toml \ --mc-type library \ diff --git a/README.md b/README.md index 50a9a99..1cf97c8 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ LadyBugDB is a required runtime dependency. A normal production install must inc ## Production install ```bash -python -m pip install codebase-graph +python -m pip install cbasegraph ``` From a repository root, run: diff --git a/conda-forge/recipe/meta.yaml b/conda-forge/recipe/meta.yaml index a91644f..157e193 100644 --- a/conda-forge/recipe/meta.yaml +++ b/conda-forge/recipe/meta.yaml @@ -1,5 +1,5 @@ {% set name = "codebase-graph" %} -{% set pypi_name = "codebase_graph" %} +{% set pypi_name = "cbasegraph" %} {% set version = "PUT_RELEASE_VERSION_HERE" %} {% set python_min = "3.10" %} @@ -8,7 +8,7 @@ package: version: {{ version }} source: - url: https://pypi.org/packages/source/{{ name[0] }}/{{ name }}/{{ pypi_name }}-{{ version }}.tar.gz + url: https://pypi.org/packages/source/{{ pypi_name[0] }}/{{ pypi_name }}/{{ pypi_name }}-{{ version }}.tar.gz sha256: PUT_RELEASE_SDIST_SHA256_HERE build: diff --git a/docs/release.md b/docs/release.md index 4f232ff..86c8d00 100644 --- a/docs/release.md +++ b/docs/release.md @@ -6,7 +6,7 @@ Configure a PyPI Trusted Publisher for: -- PyPI project: `codebase-graph` +- PyPI project: `cbasegraph` - Owner/repository: `rabii-chaarani/codebaseGraph` - Workflow: `release.yml` - Environment: `pypi` diff --git a/pyproject.toml b/pyproject.toml index 784dfcf..9a45a20 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=77", "setuptools-scm>=8", "wheel"] build-backend = "setuptools.build_meta" [project] -name = "codebase-graph" +name = "cbasegraph" dynamic = ["version"] description = "Generic codebase knowledge graph engine for Python projects." readme = "README.md" diff --git a/tests/test_release_workflows.py b/tests/test_release_workflows.py index aa284ab..01cddc5 100644 --- a/tests/test_release_workflows.py +++ b/tests/test_release_workflows.py @@ -60,6 +60,7 @@ def test_release_workflow_enforces_production_gate_before_build() -> None: def test_conda_recipe_uses_bounded_runtime_dependencies() -> None: text = Path("conda-forge/recipe/meta.yaml").read_text(encoding="utf-8") + assert '{% set pypi_name = "cbasegraph" %}' in text assert "setuptools >=77" in text assert "real-ladybug >=0.15.3,<0.16" in text assert "tomli >=2.0.1" in text @@ -73,9 +74,28 @@ def test_hosted_workflows_run_real_vulnerability_scans() -> None: for path in WORKFLOWS: text = path.read_text(encoding="utf-8") assert "pip_audit --strict" in text + assert "pip_audit --strict ." in text + assert "--skip-editable" not in text assert re.search(r"pip_audit\b[^\n]*--dry-run", text) is None +def test_supply_chain_workflow_audits_project_dependencies() -> None: + text = Path(".github/workflows/ci.yml").read_text(encoding="utf-8") + match = re.search(r" supply-chain:\n(?P.*?)(?=\n [A-Za-z0-9_-]+:|\Z)", text, re.DOTALL) + + assert match is not None + body = match.group("body") + assert 'python -m pip install ".[dev]"' in body + assert 'python -m pip install -e ".[dev]"' not in body + assert "python -m pip_audit --strict ." in body + + +def test_project_metadata_uses_published_pypi_name() -> None: + text = Path("pyproject.toml").read_text(encoding="utf-8") + + assert 'name = "cbasegraph"' in text + + def test_security_policy_exists() -> None: text = Path("SECURITY.md").read_text(encoding="utf-8") @@ -87,6 +107,7 @@ def test_security_policy_exists() -> None: def test_release_docs_list_production_confirmation_flags() -> None: text = Path("docs/release.md").read_text(encoding="utf-8") + assert "PyPI project: `cbasegraph`" in text for flag in PYPI_CONFIRMATION_FLAGS: env_var = f"CODEBASE_GRAPH_CONFIRM_{flag.upper().replace('-', '_')}" assert env_var in text From 10b21588c0b9dc5d6150fdb8c24f32aaf60defe1 Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Thu, 28 May 2026 10:14:51 +0930 Subject: [PATCH 42/53] fix: suppress GitHub Actions warnings GitHub started annotating the workflow because the pinned JavaScript actions still default to Node.js 20, and the parallel Ubuntu 3.12 CI jobs were racing to save the same setup-python pip cache key. The workflows now opt JavaScript actions into Node.js 24 explicitly and only keep pip caching on the matrix test job where cache keys are naturally separated by OS and Python version. Constraint: GitHub Actions Node.js 20 deprecation begins forcing Node.js 24 by default on June 2, 2026. Rejected: Unpin action refs to version tags | repository release gate requires immutable action pins. Rejected: Keep pip caching on all Ubuntu 3.12 jobs | setup-python emits cache reservation noise when parallel jobs share the key. Confidence: high Scope-risk: narrow Directive: Keep GitHub-hosted workflows warning-free; do not reintroduce shared setup-python cache keys across parallel jobs without checking hosted logs. Tested: .venv/bin/python -m pytest tests/test_release_workflows.py -q Tested: .venv/bin/ruff check . Tested: .venv/bin/python -m pytest -q Not-tested: Hosted GitHub Actions warning annotations before push. --- .github/workflows/ci.yml | 6 +++--- .github/workflows/release.yml | 3 +++ tests/test_release_workflows.py | 13 +++++++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 190844c..d387884 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,6 +13,9 @@ on: permissions: contents: read +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + jobs: test: name: pytest (${{ matrix.os }}, py${{ matrix.python-version }}) @@ -67,7 +70,6 @@ jobs: uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.12" - cache: pip - name: Install lint dependencies run: | @@ -92,7 +94,6 @@ jobs: uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.12" - cache: pip - name: Install package and supply-chain tools run: | @@ -137,7 +138,6 @@ jobs: uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.12" - cache: pip - name: Build distributions run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cab4c86..c557ea1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,6 +9,9 @@ on: permissions: contents: read +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + jobs: release-please: name: release please diff --git a/tests/test_release_workflows.py b/tests/test_release_workflows.py index 01cddc5..b34e5e3 100644 --- a/tests/test_release_workflows.py +++ b/tests/test_release_workflows.py @@ -123,6 +123,19 @@ def test_workflow_jobs_have_timeouts() -> None: assert missing == [] +def test_workflows_opt_javascript_actions_into_node24() -> None: + for path in WORKFLOWS: + text = path.read_text(encoding="utf-8") + + assert "FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true" in text + + +def test_ci_avoids_duplicate_pip_cache_reservation_warnings() -> None: + text = Path(".github/workflows/ci.yml").read_text(encoding="utf-8") + + assert text.count("cache: pip") == 1 + + def test_local_release_gate_passes() -> None: assert run_checks(production=False, require_conda=False, confirmations=set()) == [] From 9cbce2b9ee26916f5d8622cb81ce955f7778d7ce Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Thu, 28 May 2026 10:22:16 +0930 Subject: [PATCH 43/53] fix: remove Node 20 workflow actions The forced Node 24 environment still produced hosted warning annotations because some pinned actions continued to advertise node20 in their metadata. CI now pins checkout v5 and setup-python v6 by immutable SHA, and release-please is pinned to its Node 24 major. The artifact upload/download actions still target node20 at their latest major, so the workflows avoid them and hand release distributions to PyPI through the existing GitHub release assets instead. Constraint: Workflow action refs must remain immutable commit SHAs. Constraint: actions/upload-artifact v5 and actions/download-artifact v5 still declare runs.using node20. Rejected: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24 | GitHub still emits a warning when forced Node 24 runs node20-targeted action metadata. Rejected: Keep release artifacts as Actions artifacts | the artifact actions are themselves the remaining Node 20 warning source. Confidence: high Scope-risk: moderate Directive: Keep first-party GitHub actions pinned to commits whose action.yml declares node24, and avoid artifact actions until they publish a node24-compatible major. Tested: .venv/bin/python -m pytest tests/test_release_workflows.py -q Tested: .venv/bin/ruff check . Tested: .venv/bin/python -m pytest -q Not-tested: End-to-end production release because it requires an actual release-created run and PyPI environment approval. --- .github/workflows/ci.yml | 26 ++++++------------ .github/workflows/release.yml | 48 +++++++++++++++------------------ tests/test_release_workflows.py | 23 ++++++++++++++-- 3 files changed, 51 insertions(+), 46 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d387884..042b644 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,9 +13,6 @@ on: permissions: contents: read -env: - FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true - jobs: test: name: pytest (${{ matrix.os }}, py${{ matrix.python-version }}) @@ -37,12 +34,12 @@ jobs: steps: - name: Check out repository - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: ${{ matrix.python-version }} cache: pip @@ -62,12 +59,12 @@ jobs: steps: - name: Check out repository - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.12" @@ -86,12 +83,12 @@ jobs: steps: - name: Check out repository - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.12" @@ -116,13 +113,6 @@ jobs: --of JSON \ -o codebase-graph-sbom.cdx.json - - name: Upload SBOM - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - with: - name: codebase-graph-sbom - path: codebase-graph-sbom.cdx.json - if-no-files-found: error - package: name: package runs-on: ubuntu-latest @@ -130,12 +120,12 @@ jobs: steps: - name: Check out repository - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.12" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c557ea1..9f7310f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,9 +9,6 @@ on: permissions: contents: read -env: - FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true - jobs: release-please: name: release please @@ -28,7 +25,7 @@ jobs: steps: - name: Create release pull request or GitHub release id: release - uses: googleapis/release-please-action@5c625bfb5d1ff62eadeeb3772007f7f66fdcf071 # v4 + uses: googleapis/release-please-action@45996ed1f6d02564a971a2fa1b5860e934307cf7 # v5.0.0 with: token: ${{ secrets.RELEASE_PLEASE_TOKEN || github.token }} config-file: release-please-config.json @@ -55,7 +52,7 @@ jobs: steps: - name: Check out release tag - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: ref: ${{ env.RELEASE_TAG }} fetch-depth: 0 @@ -99,13 +96,13 @@ jobs: steps: - name: Check out release tag - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: ref: ${{ env.RELEASE_TAG }} fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.12" cache: pip @@ -185,20 +182,6 @@ jobs: /tmp/codebase-graph-sdist/bin/codebase-graph-mcp --help /tmp/codebase-graph-sdist/bin/python scripts/smoke_built_wheel.py /tmp/codebase-graph-sdist/bin/codebase-graph - - name: Upload distributions - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - with: - name: codebase-graph-${{ steps.verify-version.outputs.package-version }}-dist - path: dist/* - if-no-files-found: error - - - name: Upload SBOM artifact - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - with: - name: codebase-graph-${{ steps.verify-version.outputs.package-version }}-sbom - path: codebase-graph-${{ steps.verify-version.outputs.package-version }}-sbom.cdx.json - if-no-files-found: error - - name: Upload distributions to GitHub release env: GH_TOKEN: ${{ github.token }} @@ -216,14 +199,27 @@ jobs: name: pypi url: https://pypi.org/p/cbasegraph permissions: + contents: read id-token: write + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ needs.release-please.outputs.tag-name }} steps: - - name: Download distributions - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 - with: - name: codebase-graph-${{ needs.build.outputs.package-version }}-dist - path: dist + - name: Download distributions from GitHub release + shell: bash + run: | + mkdir -p dist + gh release download "$RELEASE_TAG" --dir dist --pattern "*.whl" --pattern "*.tar.gz" + python - <<'PY' + from pathlib import Path + + artifacts = sorted(path.name for path in Path("dist").iterdir()) + if not any(name.endswith(".whl") for name in artifacts): + raise SystemExit(f"release {artifacts=} does not include a wheel") + if not any(name.endswith(".tar.gz") for name in artifacts): + raise SystemExit(f"release {artifacts=} does not include a source distribution") + PY - name: Publish distributions to PyPI uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1 diff --git a/tests/test_release_workflows.py b/tests/test_release_workflows.py index b34e5e3..cb536ca 100644 --- a/tests/test_release_workflows.py +++ b/tests/test_release_workflows.py @@ -123,11 +123,30 @@ def test_workflow_jobs_have_timeouts() -> None: assert missing == [] -def test_workflows_opt_javascript_actions_into_node24() -> None: +def test_workflows_pin_node24_capable_first_party_actions() -> None: for path in WORKFLOWS: text = path.read_text(encoding="utf-8") - assert "FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true" in text + assert "actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd" in text + assert "actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405" in text + assert "actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5" not in text + assert "actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065" not in text + + +def test_workflows_avoid_node20_artifact_actions() -> None: + for path in WORKFLOWS: + text = path.read_text(encoding="utf-8") + + assert "actions/upload-artifact@" not in text + assert "actions/download-artifact@" not in text + + +def test_release_workflow_downloads_distributions_from_github_release() -> None: + text = Path(".github/workflows/release.yml").read_text(encoding="utf-8") + + assert 'gh release download "$RELEASE_TAG" --dir dist' in text + assert "release {artifacts=} does not include a wheel" in text + assert "release {artifacts=} does not include a source distribution" in text def test_ci_avoids_duplicate_pip_cache_reservation_warnings() -> None: From 93b9d5183a2b13e35319da24bd392ff3c0f60a7c Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Thu, 28 May 2026 10:38:13 +0930 Subject: [PATCH 44/53] fix: clear hosted workflow annotations The Node 20 action warning was fixed, but hosted checks still emitted annotations from windows-latest runner migration notices and setup-python cache deserialization. CI now uses an explicit Windows runner label and workflows no longer request pip caching, removing those hosted warning sources rather than relying on cache rollover. Constraint: The warnings are emitted by GitHub-hosted runner infrastructure, so local tests can only guard workflow shape. Rejected: Wait for stale setup-python caches to expire | the warning would continue until cache eviction and could reappear. Rejected: Keep windows-latest | GitHub currently annotates that label with a migration notice. Confidence: high Scope-risk: narrow Directive: Reintroduce setup-python caching only with hosted evidence that it does not emit cache warning annotations. Tested: .venv/bin/python -m pytest tests/test_release_workflows.py -q Tested: .venv/bin/ruff check . Tested: .venv/bin/python -m pytest -q Not-tested: Hosted Actions after this commit before push. --- .github/workflows/ci.yml | 3 +-- .github/workflows/release.yml | 1 - tests/test_release_workflows.py | 12 ++++++++++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 042b644..7b2f025 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: os: - ubuntu-latest - macos-latest - - windows-latest + - windows-2022 python-version: - "3.10" - "3.11" @@ -42,7 +42,6 @@ jobs: uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: ${{ matrix.python-version }} - cache: pip - name: Install package and test dependencies run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9f7310f..aaea1af 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -105,7 +105,6 @@ jobs: uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.12" - cache: pip - name: Build and validate distributions run: | diff --git a/tests/test_release_workflows.py b/tests/test_release_workflows.py index cb536ca..0d98f78 100644 --- a/tests/test_release_workflows.py +++ b/tests/test_release_workflows.py @@ -149,10 +149,18 @@ def test_release_workflow_downloads_distributions_from_github_release() -> None: assert "release {artifacts=} does not include a source distribution" in text -def test_ci_avoids_duplicate_pip_cache_reservation_warnings() -> None: +def test_workflows_avoid_hosted_cache_warning_annotations() -> None: + for path in WORKFLOWS: + text = path.read_text(encoding="utf-8") + + assert "cache: pip" not in text + + +def test_ci_uses_explicit_windows_runner_label() -> None: text = Path(".github/workflows/ci.yml").read_text(encoding="utf-8") - assert text.count("cache: pip") == 1 + assert "windows-2022" in text + assert "windows-latest" not in text def test_local_release_gate_passes() -> None: From c00dd8d888dae46de149766a8fbe3ef59f8d1a9b Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Thu, 28 May 2026 10:53:19 +0930 Subject: [PATCH 45/53] fix: disable workflow pip cache warnings Hosted macOS Python 3.10 setup was emitting pip cache deserialization warnings while installing certificates, even after Actions cache usage was removed. Disabling pip's on-runner cache at workflow scope keeps setup and install steps from reading the stale hosted cache while preserving the existing package/test matrix. Constraint: GitHub-hosted macOS image can contain stale pip cache entries before repository steps run. Rejected: Drop macOS Python 3.10 coverage | would hide the warning by reducing platform coverage. Confidence: medium Scope-risk: narrow Directive: Keep workflow-level pip cache disabled unless hosted runner cache annotations are explicitly rechecked. Tested: .venv/bin/python -m pytest tests/test_release_workflows.py -q Tested: .venv/bin/ruff check tests/test_release_workflows.py Tested: .venv/bin/ruff check . Tested: .venv/bin/python -m pytest -q --- .github/workflows/ci.yml | 3 +++ .github/workflows/release.yml | 3 +++ tests/test_release_workflows.py | 1 + 3 files changed, 7 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7b2f025..76db5b7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,6 +13,9 @@ on: permissions: contents: read +env: + PIP_NO_CACHE_DIR: "1" + jobs: test: name: pytest (${{ matrix.os }}, py${{ matrix.python-version }}) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index aaea1af..df7e151 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,6 +9,9 @@ on: permissions: contents: read +env: + PIP_NO_CACHE_DIR: "1" + jobs: release-please: name: release please diff --git a/tests/test_release_workflows.py b/tests/test_release_workflows.py index 0d98f78..8851802 100644 --- a/tests/test_release_workflows.py +++ b/tests/test_release_workflows.py @@ -153,6 +153,7 @@ def test_workflows_avoid_hosted_cache_warning_annotations() -> None: for path in WORKFLOWS: text = path.read_text(encoding="utf-8") + assert 'PIP_NO_CACHE_DIR: "1"' in text assert "cache: pip" not in text From be70f2ad7d17adc75d18b1db5ba1b0adc3cda0c6 Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Thu, 28 May 2026 10:57:45 +0930 Subject: [PATCH 46/53] feat: reduce agent graph-search token overhead Add an ontology-preserving block serializer, a tiktoken benchmark CLI, a SearchService fixture, focused serializer/token tests, and the generated token comparison report. Retrieval semantics and ontology definitions are left unchanged. Constraint: Do not change graph ontology or retrieval semantics Rejected: Index tables, numeric lookup dictionaries, relation abbreviations, and type abbreviations | violates the readable ontology-preserving format requirement Confidence: high Scope-risk: moderate Directive: Keep block output directly readable and preserve literal ontology terms when changing the serializer Tested: ./.venv/bin/ruff check .; ./.venv/bin/pytest -q; ./.venv/bin/python scripts/compare_graph_output_tokens.py --fixture tests/fixtures/search_service_graph_search.json --encoding o200k_base --output docs/graph_output_token_comparison.md Not-tested: Large multi-query production corpus beyond the committed SearchService fixture --- docs/graph_output_token_comparison.md | 37 +++ pyproject.toml | 4 +- scripts/compare_graph_output_tokens.py | 295 ++++++++++++++++++ src/codebase_graph/retrieval/__init__.py | 18 +- src/codebase_graph/retrieval/block_format.py | 248 +++++++++++++++ .../fixtures/search_service_graph_search.json | 117 +++++++ tests/test_graph_output_block_format.py | 109 +++++++ 7 files changed, 825 insertions(+), 3 deletions(-) create mode 100644 docs/graph_output_token_comparison.md create mode 100644 scripts/compare_graph_output_tokens.py create mode 100644 src/codebase_graph/retrieval/block_format.py create mode 100644 tests/fixtures/search_service_graph_search.json create mode 100644 tests/test_graph_output_block_format.py diff --git a/docs/graph_output_token_comparison.md b/docs/graph_output_token_comparison.md new file mode 100644 index 0000000..8f66332 --- /dev/null +++ b/docs/graph_output_token_comparison.md @@ -0,0 +1,37 @@ +# Graph Search Output Token Comparison + +## Method +- Raw format: compact JSON emitted by the current graph-search payload serializer, counted from the exact serialized JSON text with sorted keys and compact separators. +- Ontology-preserving block format: grouped `file path` blocks with readable `Class`, `Method`, `Scope`, relation, `label`, `span`, `id`, and `rank_score` terms left literal. +- Tokenizer/model: encoding `o200k_base`. +- Count method: payload-only tokens using `len(encoding.encode(text))`; chat-message wrapper tokens were not included. + +## Results +| Query | Results | Context edges | Raw tokens | Block tokens | Saved tokens | Reduction % | +|---|---:|---:|---:|---:|---:|---:| +| SearchService | 3 | 6 | 502 | 234 | 268 | 53.4% | + +## Aggregate Summary +- Samples: 1 +- Total raw tokens: 502 +- Total block tokens: 234 +- Total saved tokens: 268 +- Overall reduction: 53.4% +- Mean reduction: 53.4% +- Median reduction: 53.4% +- Min reduction: SearchService (53.4%) +- Max reduction: SearchService (53.4%) +- p90 raw/block tokens: not reported because fewer than 10 samples were compared + +## Ontology Preservation +The validator normalizes raw JSON and block output into canonical result records preserving `type`, `label`, `path`, `span`, `id`, `rank_score`, and ordered context records with `direction`, `relation`, `type`, `label`, `path`, `span`, and non-boilerplate `summary`. + +Intentional omissions: +- `results[0].context[0].summary` +- `results[1].context[0].summary` +- `results[2].context[0].summary` + +Known limitations: the block parser validates the supported graph-search fixture shape and live graph-search output shape; it is not a general-purpose parser for hand-written variants. + +## Recommendation +Use the ontology-preserving block format by default for agent-facing graph-search output when consumers need readable context. Keep JSON available for machine APIs and tests that require strict structured payloads. diff --git a/pyproject.toml b/pyproject.toml index 9a45a20..eae5dcf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ classifiers = [ [project.optional-dependencies] ladybug = [] parquet = ["pyarrow"] -dev = ["pytest", "ruff"] +dev = ["pytest", "ruff", "tiktoken"] [project.scripts] codebase-graph = "codebase_graph.cli:main" @@ -56,5 +56,5 @@ line-length = 120 target-version = "py310" [tool.pytest.ini_options] -pythonpath = ["src"] +pythonpath = ["src", "."] testpaths = ["tests"] diff --git a/scripts/compare_graph_output_tokens.py b/scripts/compare_graph_output_tokens.py new file mode 100644 index 0000000..61aeed3 --- /dev/null +++ b/scripts/compare_graph_output_tokens.py @@ -0,0 +1,295 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import statistics +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +REPO_ROOT = Path(__file__).resolve().parents[1] +SRC_ROOT = REPO_ROOT / "src" +if SRC_ROOT.as_posix() not in sys.path: + sys.path.insert(0, SRC_ROOT.as_posix()) + +from codebase_graph.mcp.runtime import runtime_config # noqa: E402 +from codebase_graph.mcp.tools import handle_tool_call # noqa: E402 +from codebase_graph.retrieval.block_format import ( # noqa: E402 + canonicalize_search_payload, + intentional_summary_omissions, + parse_search_block, + serialize_search_block, +) + + +DEFAULT_FIXTURE = REPO_ROOT / "tests" / "fixtures" / "search_service_graph_search.json" +DEFAULT_OUTPUT = REPO_ROOT / "docs" / "graph_output_token_comparison.md" + + +@dataclass(frozen=True, slots=True) +class Tokenizer: + encoding: Any + encoding_name: str + model_name: str | None + fallback_note: str = "" + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + tokenizer = resolve_tokenizer(model=args.model, encoding_name=args.encoding) + samples = _load_samples(args) + rows = [_compare_sample(sample, tokenizer) for sample in samples] + aggregate = _aggregate(rows) + _write_report(args.output, rows, aggregate, tokenizer) + _print_summary(rows, aggregate, tokenizer, args.output) + return 0 + + +def resolve_tokenizer(*, model: str | None = None, encoding_name: str | None = None) -> Tokenizer: + try: + import tiktoken + except ImportError as exc: + raise RuntimeError( + "tiktoken is required for graph output token benchmarking. Install it in the active environment " + "or run this script where tiktoken is available." + ) from exc + + if encoding_name: + encoding = tiktoken.get_encoding(encoding_name) + return Tokenizer(encoding=encoding, encoding_name=encoding.name, model_name=model) + if model: + try: + encoding = tiktoken.encoding_for_model(model) + return Tokenizer(encoding=encoding, encoding_name=encoding.name, model_name=model) + except KeyError: + encoding = tiktoken.get_encoding("o200k_base") + return Tokenizer( + encoding=encoding, + encoding_name=encoding.name, + model_name=model, + fallback_note=f"model-specific encoding unavailable for {model}; defaulted to o200k_base", + ) + encoding = tiktoken.get_encoding("o200k_base") + return Tokenizer(encoding=encoding, encoding_name=encoding.name, model_name=None) + + +def count_tokens(text: str, encoding: Any) -> int: + return len(encoding.encode(text)) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Compare graph-search JSON output with readable block output.") + parser.add_argument("--queries", action="append", default=[], help="Graph-search query to run; repeat as needed") + parser.add_argument("--fixture", action="append", type=Path, default=[], help="Path to a graph-search JSON fixture") + parser.add_argument("--model", default=None, help="Model name used to resolve a tiktoken encoding") + parser.add_argument("--encoding", default=None, help="Explicit tiktoken encoding name") + parser.add_argument("--limit", type=int, default=3, help="Graph-search result limit for live queries") + parser.add_argument("--profile", default="brief", help="Graph-search context profile for live queries") + parser.add_argument("--budget", type=int, default=600, help="Graph-search context budget for live queries") + parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT, help="Markdown report path") + parser.add_argument("--repo-root", type=Path, default=REPO_ROOT, help="Repository root for live graph-search queries") + parser.add_argument("--config", type=Path, default=None, help="Optional codebaseGraph setup config path") + parser.add_argument("--db", type=Path, default=None, help="Optional codebaseGraph database path") + parser.add_argument("--manifest", type=Path, default=None, help="Optional codebaseGraph manifest path") + parser.add_argument("--context-limit", type=int, default=2, help="Context items per result for live queries") + parser.add_argument("--detail", choices=("standard", "slim"), default="slim", help="Raw graph-search detail level") + return parser + + +def _load_samples(args: argparse.Namespace) -> list[dict[str, Any]]: + samples: list[dict[str, Any]] = [] + fixture_paths = args.fixture or ([] if args.queries else [DEFAULT_FIXTURE]) + for fixture_path in fixture_paths: + payload = json.loads(fixture_path.read_text(encoding="utf-8")) + if isinstance(payload, list): + samples.extend(_fixture_sample(item, fixture_path) for item in payload) + else: + samples.append(_fixture_sample(payload, fixture_path)) + if args.queries: + runtime = runtime_config( + repo_root=args.repo_root, + config_path=args.config, + db_path=args.db, + manifest_path=args.manifest, + ) + for query in args.queries: + payload = handle_tool_call( + "graph_search", + { + "query": query, + "limit": args.limit, + "profile": args.profile, + "budget": args.budget, + "context_limit": args.context_limit, + "detail": args.detail, + }, + runtime=runtime, + ) + samples.append({"name": query, "payload": payload, "source": "live graph-search"}) + if not samples: + raise ValueError("No samples found. Provide --queries or --fixture.") + return samples + + +def _fixture_sample(payload: dict[str, Any], fixture_path: Path) -> dict[str, Any]: + if "payload" in payload and isinstance(payload["payload"], dict): + name = str(payload.get("name") or payload["payload"].get("query") or fixture_path.stem) + return {"name": name, "payload": payload["payload"], "source": fixture_path.as_posix()} + return {"name": str(payload.get("query") or fixture_path.stem), "payload": payload, "source": fixture_path.as_posix()} + + +def _compare_sample(sample: dict[str, Any], tokenizer: Tokenizer) -> dict[str, Any]: + payload = sample["payload"] + raw_text = _raw_json(payload) + block_text = serialize_search_block(payload) + raw_canonical = canonicalize_search_payload(payload) + block_canonical = parse_search_block(block_text) + if raw_canonical != block_canonical: + raise AssertionError( + f"Block output is not semantically equivalent for {sample['name']}:\n" + f"raw={json.dumps(raw_canonical, sort_keys=True)}\n" + f"block={json.dumps(block_canonical, sort_keys=True)}" + ) + raw_tokens = count_tokens(raw_text, tokenizer.encoding) + block_tokens = count_tokens(block_text, tokenizer.encoding) + raw_chars = len(raw_text) + block_chars = len(block_text) + token_delta = raw_tokens - block_tokens + char_delta = raw_chars - block_chars + result_count = len(payload.get("results", [])) + context_edges = sum(len(result.get("context", [])) for result in payload.get("results", [])) + return { + "query": sample["name"], + "source": sample["source"], + "raw_chars": raw_chars, + "block_chars": block_chars, + "raw_tokens": raw_tokens, + "block_tokens": block_tokens, + "token_delta": token_delta, + "token_reduction_pct": _pct(token_delta, raw_tokens), + "char_reduction_pct": _pct(char_delta, raw_chars), + "results": result_count, + "context_edges": context_edges, + "tokenizer": tokenizer.encoding_name, + "model": tokenizer.model_name or "", + "intentional_omissions": intentional_summary_omissions(payload), + } + + +def _aggregate(rows: list[dict[str, Any]]) -> dict[str, Any]: + raw_tokens = [row["raw_tokens"] for row in rows] + block_tokens = [row["block_tokens"] for row in rows] + reductions = [row["token_reduction_pct"] for row in rows] + total_raw = sum(raw_tokens) + total_block = sum(block_tokens) + sorted_by_reduction = sorted(rows, key=lambda row: row["token_reduction_pct"]) + aggregate = { + "sample_count": len(rows), + "total_raw_tokens": total_raw, + "total_block_tokens": total_block, + "total_token_delta": total_raw - total_block, + "overall_token_reduction_pct": _pct(total_raw - total_block, total_raw), + "mean_token_reduction_pct": statistics.fmean(reductions) if reductions else 0.0, + "median_token_reduction_pct": statistics.median(reductions) if reductions else 0.0, + "min_reduction_case": sorted_by_reduction[0]["query"] if sorted_by_reduction else "", + "min_reduction_pct": sorted_by_reduction[0]["token_reduction_pct"] if sorted_by_reduction else 0.0, + "max_reduction_case": sorted_by_reduction[-1]["query"] if sorted_by_reduction else "", + "max_reduction_pct": sorted_by_reduction[-1]["token_reduction_pct"] if sorted_by_reduction else 0.0, + "p90_raw_tokens": None, + "p90_block_tokens": None, + } + if len(rows) >= 10: + aggregate["p90_raw_tokens"] = _p90(raw_tokens) + aggregate["p90_block_tokens"] = _p90(block_tokens) + return aggregate + + +def _write_report(path: Path, rows: list[dict[str, Any]], aggregate: dict[str, Any], tokenizer: Tokenizer) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + table_rows = "\n".join( + "| {query} | {results} | {context_edges} | {raw_tokens:,} | {block_tokens:,} | {token_delta:,} | " + "{token_reduction_pct:.1f}% |".format(**row) + for row in rows + ) + omission_lines = sorted({omission for row in rows for omission in row["intentional_omissions"]}) + omissions = "\n".join(f"- `{item}`" for item in omission_lines) or "- None" + fallback = f"\n- {tokenizer.fallback_note}" if tokenizer.fallback_note else "" + p90 = ( + f"- p90 raw/block tokens: {aggregate['p90_raw_tokens']:,} / {aggregate['p90_block_tokens']:,}\n" + if aggregate["p90_raw_tokens"] is not None + else "- p90 raw/block tokens: not reported because fewer than 10 samples were compared\n" + ) + path.write_text( + "\n".join( + [ + "# Graph Search Output Token Comparison", + "", + "## Method", + "- Raw format: compact JSON emitted by the current graph-search payload serializer, counted from the exact serialized JSON text with sorted keys and compact separators.", + "- Ontology-preserving block format: grouped `file path` blocks with readable `Class`, `Method`, `Scope`, relation, `label`, `span`, `id`, and `rank_score` terms left literal.", + f"- Tokenizer/model: encoding `{tokenizer.encoding_name}`" + + (f" resolved from model `{tokenizer.model_name}`." if tokenizer.model_name else ".") + + fallback, + "- Count method: payload-only tokens using `len(encoding.encode(text))`; chat-message wrapper tokens were not included.", + "", + "## Results", + "| Query | Results | Context edges | Raw tokens | Block tokens | Saved tokens | Reduction % |", + "|---|---:|---:|---:|---:|---:|---:|", + table_rows, + "", + "## Aggregate Summary", + f"- Samples: {aggregate['sample_count']}", + f"- Total raw tokens: {aggregate['total_raw_tokens']:,}", + f"- Total block tokens: {aggregate['total_block_tokens']:,}", + f"- Total saved tokens: {aggregate['total_token_delta']:,}", + f"- Overall reduction: {aggregate['overall_token_reduction_pct']:.1f}%", + f"- Mean reduction: {aggregate['mean_token_reduction_pct']:.1f}%", + f"- Median reduction: {aggregate['median_token_reduction_pct']:.1f}%", + f"- Min reduction: {aggregate['min_reduction_case']} ({aggregate['min_reduction_pct']:.1f}%)", + f"- Max reduction: {aggregate['max_reduction_case']} ({aggregate['max_reduction_pct']:.1f}%)", + p90.rstrip(), + "", + "## Ontology Preservation", + "The validator normalizes raw JSON and block output into canonical result records preserving `type`, `label`, `path`, `span`, `id`, `rank_score`, and ordered context records with `direction`, `relation`, `type`, `label`, `path`, `span`, and non-boilerplate `summary`.", + "", + "Intentional omissions:", + omissions, + "", + "Known limitations: the block parser validates the supported graph-search fixture shape and live graph-search output shape; it is not a general-purpose parser for hand-written variants.", + "", + "## Recommendation", + "Use the ontology-preserving block format by default for agent-facing graph-search output when consumers need readable context. Keep JSON available for machine APIs and tests that require strict structured payloads.", + "", + ] + ), + encoding="utf-8", + ) + + +def _print_summary(rows: list[dict[str, Any]], aggregate: dict[str, Any], tokenizer: Tokenizer, output_path: Path) -> None: + print(f"Compared {len(rows)} graph-search outputs using {tokenizer.encoding_name}.") + print(f"Raw: {aggregate['total_raw_tokens']:,} tokens") + print(f"Block: {aggregate['total_block_tokens']:,} tokens") + print(f"Saved: {aggregate['total_token_delta']:,} tokens") + print(f"Reduction: {aggregate['overall_token_reduction_pct']:.1f}%") + print(f"Report written to {output_path.as_posix()}") + + +def _raw_json(payload: dict[str, Any]) -> str: + return json.dumps(payload, separators=(",", ":"), sort_keys=True) + + +def _pct(delta: int | float, original: int | float) -> float: + return (float(delta) / float(original) * 100.0) if original else 0.0 + + +def _p90(values: list[int]) -> int: + ordered = sorted(values) + index = int(0.9 * (len(ordered) - 1)) + return ordered[index] + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/codebase_graph/retrieval/__init__.py b/src/codebase_graph/retrieval/__init__.py index e204c8a..801cdfd 100644 --- a/src/codebase_graph/retrieval/__init__.py +++ b/src/codebase_graph/retrieval/__init__.py @@ -1,5 +1,21 @@ """Keyword, vector, graph traversal, and ranking retrieval.""" +from .block_format import ( + canonicalize_search_payload, + intentional_summary_omissions, + parse_search_block, + serialize_search_block, +) from .search import DETAIL_LEVELS, CompactContextPayload, SearchHit, SearchRequest, SearchService -__all__ = ["DETAIL_LEVELS", "CompactContextPayload", "SearchHit", "SearchRequest", "SearchService"] +__all__ = [ + "DETAIL_LEVELS", + "CompactContextPayload", + "SearchHit", + "SearchRequest", + "SearchService", + "canonicalize_search_payload", + "intentional_summary_omissions", + "parse_search_block", + "serialize_search_block", +] diff --git a/src/codebase_graph/retrieval/block_format.py b/src/codebase_graph/retrieval/block_format.py new file mode 100644 index 0000000..68863a2 --- /dev/null +++ b/src/codebase_graph/retrieval/block_format.py @@ -0,0 +1,248 @@ +from __future__ import annotations + +import json +import re +import shlex +from typing import Any, Mapping + + +SIMPLE_VALUE_RE = re.compile(r"^[A-Za-z0-9_./:\-\[\]]+$") +SPAN_RE = re.compile(r"^L(?P\d+)-L(?P\d+)$") +ONTOLOGY_TERMS = {"Class", "Method", "Scope", "Contains", "outgoing", "path", "span", "id", "label", "rank_score"} + + +def serialize_search_block(payload: Mapping[str, Any]) -> str: + """Serialize graph-search JSON into a readable ontology-preserving block format.""" + lines = [ + " | ".join( + [ + f"q {_format_value(str(payload.get('query', '')))}", + f"budget {payload.get('budget', '')}", + f"limit {payload.get('limit', '')}", + f"profile {_format_value(str(payload.get('profile', '')))}", + ] + ) + ] + current_path: str | None = None + previous_line_was_file = False + for result in payload.get("results", []): + result_path = str(result.get("path", "")) + if result_path != current_path: + if len(lines) > 1 and not previous_line_was_file: + lines.append("") + lines.append(f"file path {_format_value(result_path)}") + current_path = result_path + previous_line_was_file = True + else: + previous_line_was_file = False + + result_span = _span(result.get("span", {})) + result_parts = [ + f"- {result.get('type', '')}", + f"label={_format_value(str(result.get('label', '')))}", + f"span={_format_span(result_span)}", + ] + if "rank_score" in result: + result_parts.append(f"rank_score={result['rank_score']}") + if "id" in result: + result_parts.append(f"id={_format_value(str(result['id']))}") + summary = _meaningful_summary(result) + if summary: + result_parts.append(f"summary={_format_value(summary)}") + lines.append(" ".join(result_parts)) + + for context in result.get("context", []): + context_path = str(context.get("path", "")) + context_span = _span(context.get("span", {})) + span_text = "L=same" if context_span == result_span else _format_span(context_span) + context_parts = [ + f" {context.get('direction', '')}", + str(context.get("relation", "")), + str(context.get("type", "")), + f"label={_format_value(str(context.get('label', '')))}", + ] + if context_path and context_path != current_path: + context_parts.append(f"path={_format_value(context_path)}") + context_parts.append(f"span={span_text}") + context_summary = _meaningful_summary(context) + if context_summary: + context_parts.append(f"summary={_format_value(context_summary)}") + lines.append(" ".join(context_parts)) + previous_line_was_file = False + return "\n".join(lines) + "\n" + + +def canonicalize_search_payload(payload: Mapping[str, Any]) -> dict[str, Any]: + records: list[dict[str, Any]] = [] + for result in payload.get("results", []): + result_record = { + "type": result.get("type", ""), + "label": result.get("label", ""), + "path": result.get("path", ""), + "span": _span(result.get("span", {})), + "id": result.get("id", ""), + "rank_score": result.get("rank_score"), + "context": [], + } + result_summary = _meaningful_summary(result) + if result_summary: + result_record["summary"] = result_summary + for context in result.get("context", []): + context_record = { + "direction": context.get("direction", ""), + "relation": context.get("relation", ""), + "type": context.get("type", ""), + "label": context.get("label", ""), + "path": context.get("path", ""), + "span": _span(context.get("span", {})), + } + context_summary = _meaningful_summary(context) + if context_summary: + context_record["summary"] = context_summary + result_record["context"].append(context_record) + records.append(result_record) + return {"results": records} + + +def parse_search_block(text: str) -> dict[str, Any]: + records: list[dict[str, Any]] = [] + current_path = "" + current_result: dict[str, Any] | None = None + for raw_line in text.splitlines(): + if not raw_line.strip() or raw_line.startswith("q "): + continue + if raw_line.startswith("file path "): + current_path = _parse_value(raw_line[len("file path ") :]) + current_result = None + continue + if raw_line.startswith("- "): + tokens = shlex.split(raw_line) + fields = _keyed_fields(tokens[2:]) + current_result = { + "type": tokens[1], + "label": fields.get("label", ""), + "path": current_path, + "span": _parse_span(fields.get("span", "")), + "id": fields.get("id", ""), + "rank_score": _parse_number(fields.get("rank_score")), + "context": [], + } + if fields.get("summary"): + current_result["summary"] = fields["summary"] + records.append(current_result) + continue + if raw_line.startswith(" "): + if current_result is None: + raise ValueError(f"Context line has no parent result: {raw_line}") + tokens = shlex.split(raw_line.strip()) + fields = _keyed_fields(tokens[3:]) + span = current_result["span"] if fields.get("span") == "L=same" else _parse_span(fields.get("span", "")) + context_record = { + "direction": tokens[0], + "relation": tokens[1], + "type": tokens[2], + "label": fields.get("label", ""), + "path": fields.get("path", current_path), + "span": span, + } + if fields.get("summary"): + context_record["summary"] = fields["summary"] + current_result["context"].append(context_record) + continue + raise ValueError(f"Unknown block line: {raw_line}") + return {"results": records} + + +def intentional_summary_omissions(payload: Mapping[str, Any]) -> list[str]: + omissions: list[str] = [] + for result_index, result in enumerate(payload.get("results", [])): + if _is_boilerplate_summary(result): + omissions.append(f"results[{result_index}].summary") + for context_index, context in enumerate(result.get("context", [])): + if _is_boilerplate_summary(context): + omissions.append(f"results[{result_index}].context[{context_index}].summary") + return omissions + + +def _keyed_fields(tokens: list[str]) -> dict[str, str]: + fields: dict[str, str] = {} + for token in tokens: + if "=" not in token: + continue + key, value = token.split("=", 1) + fields[key] = value + return fields + + +def _format_value(value: str) -> str: + if value and SIMPLE_VALUE_RE.match(value): + return value + return json.dumps(value, ensure_ascii=True) + + +def _parse_value(value: str) -> str: + if value.startswith('"'): + return str(json.loads(value)) + return value + + +def _span(value: Any) -> dict[str, int]: + if not isinstance(value, Mapping): + return {} + span: dict[str, int] = {} + if value.get("line_start") is not None: + span["line_start"] = int(value["line_start"]) + if value.get("line_end") is not None: + span["line_end"] = int(value["line_end"]) + return span + + +def _format_span(span: Mapping[str, int]) -> str: + start = span.get("line_start") + end = span.get("line_end") + if start is None or end is None: + return "L?" + return f"L{start}-L{end}" + + +def _parse_span(value: str) -> dict[str, int]: + match = SPAN_RE.match(value) + if not match: + return {} + return {"line_start": int(match.group("start")), "line_end": int(match.group("end"))} + + +def _parse_number(value: str | None) -> int | float | None: + if value is None: + return None + try: + as_float = float(value) + except ValueError: + return None + return int(as_float) if as_float.is_integer() else as_float + + +def _meaningful_summary(record: Mapping[str, Any]) -> str: + summary = str(record.get("summary", "")) + return "" if _is_boilerplate_summary(record) else summary + + +def _is_boilerplate_summary(record: Mapping[str, Any]) -> bool: + summary = str(record.get("summary", "")) + label = str(record.get("label", "")) + node_type = str(record.get("type", "")) + if not summary or summary == label: + return bool(summary) + if node_type == "Scope" and label.endswith(" scope"): + scoped_label = label[: -len(" scope")] + return summary == f"Scope for {scoped_label}" + return False + + +__all__ = [ + "ONTOLOGY_TERMS", + "canonicalize_search_payload", + "intentional_summary_omissions", + "parse_search_block", + "serialize_search_block", +] diff --git a/tests/fixtures/search_service_graph_search.json b/tests/fixtures/search_service_graph_search.json new file mode 100644 index 0000000..4c0b601 --- /dev/null +++ b/tests/fixtures/search_service_graph_search.json @@ -0,0 +1,117 @@ +{ + "budget": 600, + "limit": 3, + "profile": "brief", + "query": "SearchService", + "results": [ + { + "context": [ + { + "direction": "outgoing", + "label": "SearchService scope", + "path": "src/codebase_graph/retrieval/search.py", + "relation": "Contains", + "span": { + "line_end": 173, + "line_start": 117 + }, + "summary": "Scope for SearchService", + "type": "Scope" + }, + { + "direction": "outgoing", + "label": "__init__", + "path": "src/codebase_graph/retrieval/search.py", + "relation": "Contains", + "span": { + "line_end": 121, + "line_start": 118 + }, + "type": "Method" + } + ], + "id": "Class:943d6556d328f1c7ca67", + "label": "SearchService", + "path": "src/codebase_graph/retrieval/search.py", + "rank_score": 1.351608, + "span": { + "line_end": 173, + "line_start": 117 + }, + "type": "Class" + }, + { + "context": [ + { + "direction": "outgoing", + "label": "__init__ scope", + "path": "src/codebase_graph/retrieval/search.py", + "relation": "Contains", + "span": { + "line_end": 121, + "line_start": 118 + }, + "summary": "Scope for __init__", + "type": "Scope" + }, + { + "direction": "outgoing", + "label": "self.store", + "path": "src/codebase_graph/retrieval/search.py", + "relation": "Contains", + "span": { + "line_end": 119, + "line_start": 119 + }, + "summary": "Stores the graph backend for later search calls.", + "type": "InstanceAttribute" + } + ], + "id": "Method:3c775c9656a4d6b85843", + "label": "__init__", + "path": "src/codebase_graph/retrieval/search.py", + "rank_score": 1.047561, + "span": { + "line_end": 121, + "line_start": 118 + }, + "type": "Method" + }, + { + "context": [ + { + "direction": "outgoing", + "label": "search scope", + "path": "src/codebase_graph/retrieval/search.py", + "relation": "Contains", + "span": { + "line_end": 149, + "line_start": 123 + }, + "summary": "Scope for search", + "type": "Scope" + }, + { + "direction": "outgoing", + "label": "list[SearchHit]", + "path": "src/codebase_graph/retrieval/search.py", + "relation": "Contains", + "span": { + "line_end": 132, + "line_start": 132 + }, + "type": "TypeAnnotation" + } + ], + "id": "Method:9a6ff8b159b17320c004", + "label": "search", + "path": "src/codebase_graph/retrieval/search.py", + "rank_score": 1.047561, + "span": { + "line_end": 149, + "line_start": 123 + }, + "type": "Method" + } + ] +} diff --git a/tests/test_graph_output_block_format.py b/tests/test_graph_output_block_format.py new file mode 100644 index 0000000..6437812 --- /dev/null +++ b/tests/test_graph_output_block_format.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path +from typing import Any + +from codebase_graph.retrieval.block_format import ( + ONTOLOGY_TERMS, + canonicalize_search_payload, + parse_search_block, + serialize_search_block, +) + + +FIXTURE_PATH = Path(__file__).parent / "fixtures" / "search_service_graph_search.json" +SCRIPT_PATH = Path(__file__).parents[1] / "scripts" / "compare_graph_output_tokens.py" + + +class _WhitespaceEncoding: + def encode(self, text: str) -> list[str]: + return text.split() + + +def test_token_counting_uses_encoded_text_length() -> None: + module = _load_benchmark_script() + + assert module.count_tokens("Class SearchService Method", _WhitespaceEncoding()) == 3 + + +def test_raw_vs_block_comparison_preserves_search_service_fixture() -> None: + payload = json.loads(FIXTURE_PATH.read_text(encoding="utf-8")) + block = serialize_search_block(payload) + + assert parse_search_block(block) == canonicalize_search_payload(payload) + + +def test_l_same_is_only_emitted_for_matching_context_spans() -> None: + payload = { + "query": "SearchService", + "profile": "brief", + "limit": 1, + "budget": 600, + "results": [ + { + "id": "Class:1", + "type": "Class", + "label": "SearchService", + "path": "src/codebase_graph/retrieval/search.py", + "span": {"line_start": 10, "line_end": 20}, + "rank_score": 1.0, + "context": [ + { + "direction": "outgoing", + "relation": "Contains", + "type": "Scope", + "label": "SearchService scope", + "path": "src/codebase_graph/retrieval/search.py", + "span": {"line_start": 10, "line_end": 20}, + "summary": "Scope for SearchService", + }, + { + "direction": "outgoing", + "relation": "Contains", + "type": "Method", + "label": "__init__", + "path": "src/codebase_graph/retrieval/search.py", + "span": {"line_start": 11, "line_end": 12}, + }, + ], + } + ], + } + + block = serialize_search_block(payload) + + assert block.count("span=L=same") == 1 + assert "Method label=__init__ span=L11-L12" in block + + +def test_non_boilerplate_context_summaries_are_preserved() -> None: + payload = json.loads(FIXTURE_PATH.read_text(encoding="utf-8")) + block = serialize_search_block(payload) + + assert 'summary="Stores the graph backend for later search calls."' in block + assert parse_search_block(block) == canonicalize_search_payload(payload) + + +def test_block_format_keeps_ontology_terms_literal() -> None: + payload = json.loads(FIXTURE_PATH.read_text(encoding="utf-8")) + block = serialize_search_block(payload) + + for term in ONTOLOGY_TERMS: + assert term in block + assert "rank_score=" in block + assert "label=" in block + assert "span=" in block + assert "path " in block + + +def _load_benchmark_script() -> Any: + spec = importlib.util.spec_from_file_location("compare_graph_output_tokens", SCRIPT_PATH) + assert spec is not None + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module From 889aaaf953e79ef66e4b06b24d934b9f72301ca8 Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Thu, 28 May 2026 11:19:43 +0930 Subject: [PATCH 47/53] feat: add reduced agent graph block benchmark Add a display-only agent block serializer and a benchmark flag so token comparisons can measure the aggressive format separately from the exact ontology-preserving block. Constraint: Preserve the original ontology-preserving serializer for exact semantic-equivalence use cases Rejected: Replacing the ontology-preserving block outright | stable IDs and exact scores are still needed for machine-facing graph workflows Confidence: high Scope-risk: narrow Directive: Use --block-format ontology when canonical equivalence matters; --block-format agent is intentionally lossy Tested: ./.venv/bin/ruff check .; ./.venv/bin/pytest -q; ./.venv/bin/python scripts/compare_graph_output_tokens.py --queries SearchService --queries CompactContextBuilder --queries GraphMaterializer --queries handle_tool_call --queries schema_payload --queries GraphRuntimeConfig --queries ContextNode --queries SearchRequest --queries LadybugCodeGraphStore --queries graph_query_adapter --encoding o200k_base --limit 3 --profile brief --budget 600 --context-limit 2 --block-format agent --output /tmp/graph_output_token_comparison_10_examples_agent_reduced.md Not-tested: Agent reduced-mode parser or canonical equality, because this mode intentionally omits fields --- scripts/compare_graph_output_tokens.py | 62 +++++++++++++---- src/codebase_graph/retrieval/__init__.py | 2 + src/codebase_graph/retrieval/block_format.py | 71 ++++++++++++++++++++ tests/test_graph_output_block_format.py | 22 ++++++ 4 files changed, 143 insertions(+), 14 deletions(-) diff --git a/scripts/compare_graph_output_tokens.py b/scripts/compare_graph_output_tokens.py index 61aeed3..3675151 100644 --- a/scripts/compare_graph_output_tokens.py +++ b/scripts/compare_graph_output_tokens.py @@ -20,6 +20,7 @@ canonicalize_search_payload, intentional_summary_omissions, parse_search_block, + serialize_agent_search_block, serialize_search_block, ) @@ -40,7 +41,7 @@ def main(argv: list[str] | None = None) -> int: args = _parser().parse_args(argv) tokenizer = resolve_tokenizer(model=args.model, encoding_name=args.encoding) samples = _load_samples(args) - rows = [_compare_sample(sample, tokenizer) for sample in samples] + rows = [_compare_sample(sample, tokenizer, block_format=args.block_format) for sample in samples] aggregate = _aggregate(rows) _write_report(args.output, rows, aggregate, tokenizer) _print_summary(rows, aggregate, tokenizer, args.output) @@ -95,6 +96,12 @@ def _parser() -> argparse.ArgumentParser: parser.add_argument("--manifest", type=Path, default=None, help="Optional codebaseGraph manifest path") parser.add_argument("--context-limit", type=int, default=2, help="Context items per result for live queries") parser.add_argument("--detail", choices=("standard", "slim"), default="slim", help="Raw graph-search detail level") + parser.add_argument( + "--block-format", + choices=("ontology", "agent"), + default="ontology", + help="Block serializer to compare against raw JSON", + ) return parser @@ -140,18 +147,21 @@ def _fixture_sample(payload: dict[str, Any], fixture_path: Path) -> dict[str, An return {"name": str(payload.get("query") or fixture_path.stem), "payload": payload, "source": fixture_path.as_posix()} -def _compare_sample(sample: dict[str, Any], tokenizer: Tokenizer) -> dict[str, Any]: +def _compare_sample(sample: dict[str, Any], tokenizer: Tokenizer, *, block_format: str) -> dict[str, Any]: payload = sample["payload"] raw_text = _raw_json(payload) - block_text = serialize_search_block(payload) - raw_canonical = canonicalize_search_payload(payload) - block_canonical = parse_search_block(block_text) - if raw_canonical != block_canonical: - raise AssertionError( - f"Block output is not semantically equivalent for {sample['name']}:\n" - f"raw={json.dumps(raw_canonical, sort_keys=True)}\n" - f"block={json.dumps(block_canonical, sort_keys=True)}" - ) + if block_format == "agent": + block_text = serialize_agent_search_block(payload) + else: + block_text = serialize_search_block(payload) + raw_canonical = canonicalize_search_payload(payload) + block_canonical = parse_search_block(block_text) + if raw_canonical != block_canonical: + raise AssertionError( + f"Block output is not semantically equivalent for {sample['name']}:\n" + f"raw={json.dumps(raw_canonical, sort_keys=True)}\n" + f"block={json.dumps(block_canonical, sort_keys=True)}" + ) raw_tokens = count_tokens(raw_text, tokenizer.encoding) block_tokens = count_tokens(block_text, tokenizer.encoding) raw_chars = len(raw_text) @@ -174,6 +184,7 @@ def _compare_sample(sample: dict[str, Any], tokenizer: Tokenizer) -> dict[str, A "context_edges": context_edges, "tokenizer": tokenizer.encoding_name, "model": tokenizer.model_name or "", + "block_format": block_format, "intentional_omissions": intentional_summary_omissions(payload), } @@ -208,6 +219,7 @@ def _aggregate(rows: list[dict[str, Any]]) -> dict[str, Any]: def _write_report(path: Path, rows: list[dict[str, Any]], aggregate: dict[str, Any], tokenizer: Tokenizer) -> None: path.parent.mkdir(parents=True, exist_ok=True) + block_format = rows[0].get("block_format", "ontology") if rows else "ontology" table_rows = "\n".join( "| {query} | {results} | {context_edges} | {raw_tokens:,} | {block_tokens:,} | {token_delta:,} | " "{token_reduction_pct:.1f}% |".format(**row) @@ -221,6 +233,28 @@ def _write_report(path: Path, rows: list[dict[str, Any]], aggregate: dict[str, A if aggregate["p90_raw_tokens"] is not None else "- p90 raw/block tokens: not reported because fewer than 10 samples were compared\n" ) + if block_format == "agent": + format_description = "- Reduced agent block format: grouped `file path` blocks with query settings, IDs, boilerplate same-span scope context, duplicate child-result edges, low-value type annotations, and excess score precision removed." + preservation_text = ( + "This reduced display mode intentionally does not preserve every raw JSON field. It keeps the fields most useful " + "for code navigation and follow-up inspection: file path, ordered result type/label/span, rounded `rank_score`, " + "and non-boilerplate context with summaries. Use `--block-format ontology` when exact canonical equivalence is required." + ) + recommendation = ( + "Use the reduced agent block when the consumer is an interactive coding agent optimizing for quick navigation. " + "Use the ontology-preserving block or raw JSON when stable IDs, exact scores, or complete context records are required." + ) + else: + format_description = "- Ontology-preserving block format: grouped `file path` blocks with readable `Class`, `Method`, `Scope`, relation, `label`, `span`, `id`, and `rank_score` terms left literal." + preservation_text = ( + "The validator normalizes raw JSON and block output into canonical result records preserving `type`, `label`, " + "`path`, `span`, `id`, `rank_score`, and ordered context records with `direction`, `relation`, `type`, " + "`label`, `path`, `span`, and non-boilerplate `summary`." + ) + recommendation = ( + "Use the ontology-preserving block format by default for agent-facing graph-search output when consumers need " + "readable context. Keep JSON available for machine APIs and tests that require strict structured payloads." + ) path.write_text( "\n".join( [ @@ -228,7 +262,7 @@ def _write_report(path: Path, rows: list[dict[str, Any]], aggregate: dict[str, A "", "## Method", "- Raw format: compact JSON emitted by the current graph-search payload serializer, counted from the exact serialized JSON text with sorted keys and compact separators.", - "- Ontology-preserving block format: grouped `file path` blocks with readable `Class`, `Method`, `Scope`, relation, `label`, `span`, `id`, and `rank_score` terms left literal.", + format_description, f"- Tokenizer/model: encoding `{tokenizer.encoding_name}`" + (f" resolved from model `{tokenizer.model_name}`." if tokenizer.model_name else ".") + fallback, @@ -252,7 +286,7 @@ def _write_report(path: Path, rows: list[dict[str, Any]], aggregate: dict[str, A p90.rstrip(), "", "## Ontology Preservation", - "The validator normalizes raw JSON and block output into canonical result records preserving `type`, `label`, `path`, `span`, `id`, `rank_score`, and ordered context records with `direction`, `relation`, `type`, `label`, `path`, `span`, and non-boilerplate `summary`.", + preservation_text, "", "Intentional omissions:", omissions, @@ -260,7 +294,7 @@ def _write_report(path: Path, rows: list[dict[str, Any]], aggregate: dict[str, A "Known limitations: the block parser validates the supported graph-search fixture shape and live graph-search output shape; it is not a general-purpose parser for hand-written variants.", "", "## Recommendation", - "Use the ontology-preserving block format by default for agent-facing graph-search output when consumers need readable context. Keep JSON available for machine APIs and tests that require strict structured payloads.", + recommendation, "", ] ), diff --git a/src/codebase_graph/retrieval/__init__.py b/src/codebase_graph/retrieval/__init__.py index 801cdfd..09fe35e 100644 --- a/src/codebase_graph/retrieval/__init__.py +++ b/src/codebase_graph/retrieval/__init__.py @@ -4,6 +4,7 @@ canonicalize_search_payload, intentional_summary_omissions, parse_search_block, + serialize_agent_search_block, serialize_search_block, ) from .search import DETAIL_LEVELS, CompactContextPayload, SearchHit, SearchRequest, SearchService @@ -17,5 +18,6 @@ "canonicalize_search_payload", "intentional_summary_omissions", "parse_search_block", + "serialize_agent_search_block", "serialize_search_block", ] diff --git a/src/codebase_graph/retrieval/block_format.py b/src/codebase_graph/retrieval/block_format.py index 68863a2..d75b284 100644 --- a/src/codebase_graph/retrieval/block_format.py +++ b/src/codebase_graph/retrieval/block_format.py @@ -72,6 +72,53 @@ def serialize_search_block(payload: Mapping[str, Any]) -> str: return "\n".join(lines) + "\n" +def serialize_agent_search_block(payload: Mapping[str, Any]) -> str: + """Serialize graph-search JSON into a more aggressive display-only agent block.""" + lines = [f"q {_format_value(str(payload.get('query', '')))}"] + current_path: str | None = None + result_keys = {_record_key(result) for result in payload.get("results", [])} + for result in payload.get("results", []): + result_path = str(result.get("path", "")) + if result_path != current_path: + if len(lines) > 1: + lines.append("") + lines.append(f"file path {_format_value(result_path)}") + current_path = result_path + + result_span = _span(result.get("span", {})) + result_parts = [ + f"- {result.get('type', '')}", + _format_value(str(result.get("label", ""))), + _format_span(result_span), + ] + if "rank_score" in result: + result_parts.append(f"rank_score={float(result['rank_score']):.2f}") + summary = _meaningful_summary(result) + if summary: + result_parts.append(f"summary={_format_value(summary)}") + lines.append(" ".join(result_parts)) + + for context in result.get("context", []): + if _omit_agent_context(context, parent_span=result_span, result_keys=result_keys): + continue + context_path = str(context.get("path", "")) + context_span = _span(context.get("span", {})) + context_parts = [ + f" {context.get('direction', '')}", + str(context.get("relation", "")), + str(context.get("type", "")), + _format_value(str(context.get("label", ""))), + _format_span(context_span), + ] + if context_path and context_path != current_path: + context_parts.append(f"path={_format_value(context_path)}") + context_summary = _meaningful_summary(context) + if context_summary: + context_parts.append(f"summary={_format_value(context_summary)}") + lines.append(" ".join(context_parts)) + return "\n".join(lines) + "\n" + + def canonicalize_search_payload(payload: Mapping[str, Any]) -> dict[str, Any]: records: list[dict[str, Any]] = [] for result in payload.get("results", []): @@ -239,10 +286,34 @@ def _is_boilerplate_summary(record: Mapping[str, Any]) -> bool: return False +def _omit_agent_context( + context: Mapping[str, Any], + *, + parent_span: Mapping[str, int], + result_keys: set[tuple[str, str, str, tuple[tuple[str, int], ...]]], +) -> bool: + context_span = _span(context.get("span", {})) + if _is_boilerplate_summary(context) and context_span == dict(parent_span): + return True + if _record_key(context) in result_keys: + return True + return context.get("type") == "TypeAnnotation" + + +def _record_key(record: Mapping[str, Any]) -> tuple[str, str, str, tuple[tuple[str, int], ...]]: + return ( + str(record.get("type", "")), + str(record.get("label", "")), + str(record.get("path", "")), + tuple(sorted(_span(record.get("span", {})).items())), + ) + + __all__ = [ "ONTOLOGY_TERMS", "canonicalize_search_payload", "intentional_summary_omissions", "parse_search_block", + "serialize_agent_search_block", "serialize_search_block", ] diff --git a/tests/test_graph_output_block_format.py b/tests/test_graph_output_block_format.py index 6437812..3da13b0 100644 --- a/tests/test_graph_output_block_format.py +++ b/tests/test_graph_output_block_format.py @@ -10,6 +10,7 @@ ONTOLOGY_TERMS, canonicalize_search_payload, parse_search_block, + serialize_agent_search_block, serialize_search_block, ) @@ -99,6 +100,27 @@ def test_block_format_keeps_ontology_terms_literal() -> None: assert "path " in block +def test_agent_block_reduces_display_only_boilerplate() -> None: + payload = json.loads(FIXTURE_PATH.read_text(encoding="utf-8")) + block = serialize_agent_search_block(payload) + + assert "q SearchService\n" in block + assert "budget" not in block + assert "limit" not in block + assert "profile" not in block + assert "id=" not in block + assert "rank_score=1.35" in block + assert "rank_score=1.351608" not in block + assert "SearchService scope" not in block + assert "search scope" not in block + assert "outgoing Contains Method __init__" not in block + assert "TypeAnnotation" not in block + assert ( + 'outgoing Contains InstanceAttribute self.store L119-L119 ' + 'summary="Stores the graph backend for later search calls."' + ) in block + + def _load_benchmark_script() -> Any: spec = importlib.util.spec_from_file_location("compare_graph_output_tokens", SCRIPT_PATH) assert spec is not None From 76ffa03756de3e9c17818cec1d3cad057722552e Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Thu, 28 May 2026 11:25:49 +0930 Subject: [PATCH 48/53] feat: retain IDs in reduced graph blocks Restore stable result IDs to the reduced agent-facing block serializer so follow-up graph calls can still target exact nodes while keeping the other display-only reductions. Constraint: Keep the reduced block readable and preserve the exact ontology-preserving serializer unchanged Rejected: Leaving IDs omitted | saves tokens but weakens follow-up graph lookup workflows Confidence: high Scope-risk: narrow Directive: Result IDs are intentionally retained in both ontology and agent block formats Tested: ./.venv/bin/ruff check .; ./.venv/bin/pytest -q; ./.venv/bin/python scripts/compare_graph_output_tokens.py --queries SearchService --queries CompactContextBuilder --queries GraphMaterializer --queries handle_tool_call --queries schema_payload --queries GraphRuntimeConfig --queries ContextNode --queries SearchRequest --queries LadybugCodeGraphStore --queries graph_query_adapter --encoding o200k_base --limit 3 --profile brief --budget 600 --context-limit 2 --block-format agent --output /tmp/graph_output_token_comparison_10_examples_agent_ids.md Not-tested: No additional corpus beyond the 10 live benchmark queries --- scripts/compare_graph_output_tokens.py | 4 ++-- src/codebase_graph/retrieval/block_format.py | 2 ++ tests/test_graph_output_block_format.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/compare_graph_output_tokens.py b/scripts/compare_graph_output_tokens.py index 3675151..24c5098 100644 --- a/scripts/compare_graph_output_tokens.py +++ b/scripts/compare_graph_output_tokens.py @@ -234,10 +234,10 @@ def _write_report(path: Path, rows: list[dict[str, Any]], aggregate: dict[str, A else "- p90 raw/block tokens: not reported because fewer than 10 samples were compared\n" ) if block_format == "agent": - format_description = "- Reduced agent block format: grouped `file path` blocks with query settings, IDs, boilerplate same-span scope context, duplicate child-result edges, low-value type annotations, and excess score precision removed." + format_description = "- Reduced agent block format: grouped `file path` blocks with stable result IDs retained and query settings, boilerplate same-span scope context, duplicate child-result edges, low-value type annotations, and excess score precision removed." preservation_text = ( "This reduced display mode intentionally does not preserve every raw JSON field. It keeps the fields most useful " - "for code navigation and follow-up inspection: file path, ordered result type/label/span, rounded `rank_score`, " + "for code navigation and follow-up inspection: file path, stable result `id`, ordered result type/label/span, rounded `rank_score`, " "and non-boilerplate context with summaries. Use `--block-format ontology` when exact canonical equivalence is required." ) recommendation = ( diff --git a/src/codebase_graph/retrieval/block_format.py b/src/codebase_graph/retrieval/block_format.py index d75b284..d4c15d6 100644 --- a/src/codebase_graph/retrieval/block_format.py +++ b/src/codebase_graph/retrieval/block_format.py @@ -93,6 +93,8 @@ def serialize_agent_search_block(payload: Mapping[str, Any]) -> str: ] if "rank_score" in result: result_parts.append(f"rank_score={float(result['rank_score']):.2f}") + if "id" in result: + result_parts.append(f"id={_format_value(str(result['id']))}") summary = _meaningful_summary(result) if summary: result_parts.append(f"summary={_format_value(summary)}") diff --git a/tests/test_graph_output_block_format.py b/tests/test_graph_output_block_format.py index 3da13b0..77ac6fd 100644 --- a/tests/test_graph_output_block_format.py +++ b/tests/test_graph_output_block_format.py @@ -108,7 +108,8 @@ def test_agent_block_reduces_display_only_boilerplate() -> None: assert "budget" not in block assert "limit" not in block assert "profile" not in block - assert "id=" not in block + assert "id=Class:943d6556d328f1c7ca67" in block + assert "id=Method:3c775c9656a4d6b85843" in block assert "rank_score=1.35" in block assert "rank_score=1.351608" not in block assert "SearchService scope" not in block From 9a7a41f40531ebacbe9f3a0f244813b3be06ff2a Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Thu, 28 May 2026 11:58:23 +0930 Subject: [PATCH 49/53] test: add PyPI environment smoke check The release workflow could only exercise the pypi environment during a real release, which made Trusted Publishing setup hard to validate without uploading an immutable package version. Add a workflow_dispatch smoke path that enters the same pypi environment and validates GitHub OIDC claims for release.yml while skipping release-please and all publish steps. Constraint: PyPI uploads are immutable and should not be used as environment smoke tests. Rejected: Publish a throwaway production version | would permanently consume a real PyPI version. Rejected: Separate smoke workflow | would not validate the release.yml workflow identity PyPI is configured to trust. Confidence: medium Scope-risk: narrow Directive: Keep this smoke path non-publishing; use the real release path for upload validation. Tested: .venv/bin/python -m pytest tests/test_release_workflows.py -q Tested: .venv/bin/python scripts/check_release_gate.py --production --confirm trusted-publisher --confirm pypi-environment --confirm hosted-ci-green --confirm private-vulnerability-reporting Tested: .venv/bin/ruff check tests/test_release_workflows.py Tested: .venv/bin/ruff check . Tested: .venv/bin/python -m pytest -q --- .github/workflows/release.yml | 64 +++++++++++++++++++++++++++++++++ tests/test_release_workflows.py | 19 ++++++++++ 2 files changed, 83 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index df7e151..a293ceb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,6 +5,12 @@ on: branches: - main workflow_dispatch: + inputs: + pypi-environment-smoke: + description: Verify the pypi environment and OIDC claims without publishing. + required: false + type: boolean + default: false permissions: contents: read @@ -13,8 +19,66 @@ env: PIP_NO_CACHE_DIR: "1" jobs: + pypi-environment-smoke: + name: pypi environment smoke + if: ${{ github.event_name == 'workflow_dispatch' && inputs.pypi-environment-smoke }} + runs-on: ubuntu-latest + timeout-minutes: 10 + environment: + name: pypi + permissions: + contents: read + id-token: write + + steps: + - name: Verify PyPI environment OIDC claims + shell: bash + run: | + python - <<'PY' + import base64 + import json + import os + import urllib.request + + request_url = os.environ["ACTIONS_ID_TOKEN_REQUEST_URL"] + separator = "&" if "?" in request_url else "?" + request = urllib.request.Request( + f"{request_url}{separator}audience=pypi", + headers={"Authorization": f"bearer {os.environ['ACTIONS_ID_TOKEN_REQUEST_TOKEN']}"}, + ) + with urllib.request.urlopen(request, timeout=30) as response: + token = json.load(response)["value"] + + payload = token.split(".")[1] + payload += "=" * (-len(payload) % 4) + claims = json.loads(base64.urlsafe_b64decode(payload)) + + expected_claims = { + "aud": "pypi", + "environment": "pypi", + "repository": os.environ["GITHUB_REPOSITORY"], + "repository_owner": os.environ["GITHUB_REPOSITORY_OWNER"], + "workflow": "Release", + } + for claim, expected in expected_claims.items(): + actual = claims.get(claim) + if actual != expected: + raise SystemExit(f"OIDC claim {claim!r} expected {expected!r}, got {actual!r}") + + expected_workflow_ref_prefix = f"{os.environ['GITHUB_REPOSITORY']}/.github/workflows/release.yml@" + workflow_ref = claims.get("workflow_ref") + if not isinstance(workflow_ref, str) or not workflow_ref.startswith(expected_workflow_ref_prefix): + raise SystemExit( + f"OIDC claim 'workflow_ref' expected to start with {expected_workflow_ref_prefix!r}, " + f"got {workflow_ref!r}" + ) + + print("pypi environment OIDC claims verified") + PY + release-please: name: release please + if: ${{ !inputs.pypi-environment-smoke }} runs-on: ubuntu-latest timeout-minutes: 10 permissions: diff --git a/tests/test_release_workflows.py b/tests/test_release_workflows.py index 8851802..7217620 100644 --- a/tests/test_release_workflows.py +++ b/tests/test_release_workflows.py @@ -57,6 +57,25 @@ def test_release_workflow_enforces_production_gate_before_build() -> None: assert "build:\n name: build release distributions\n needs:\n - release-please\n - production-gate" in text +def test_release_workflow_can_smoke_test_pypi_environment_without_publishing() -> None: + text = Path(".github/workflows/release.yml").read_text(encoding="utf-8") + + assert "pypi-environment-smoke:" in text + assert "github.event_name == 'workflow_dispatch' && inputs.pypi-environment-smoke" in text + assert "name: pypi" in text + assert "id-token: write" in text + assert "audience=pypi" in text + assert '"environment": "pypi"' in text + assert ".github/workflows/release.yml@" in text + assert "pypi environment OIDC claims verified" in text + + +def test_release_please_is_skipped_during_pypi_environment_smoke() -> None: + text = Path(".github/workflows/release.yml").read_text(encoding="utf-8") + + assert "release-please:\n name: release please\n if: ${{ !inputs.pypi-environment-smoke }}" in text + + def test_conda_recipe_uses_bounded_runtime_dependencies() -> None: text = Path("conda-forge/recipe/meta.yaml").read_text(encoding="utf-8") From 8487f9fbab55569e88b60141a14a71e805be58d9 Mon Sep 17 00:00:00 2001 From: Rabii Chaarani <50892556+rabii-chaarani@users.noreply.github.com> Date: Thu, 28 May 2026 13:02:23 +0930 Subject: [PATCH 50/53] feat: new compact graph context block (#1) * feat: expose block graph output to agents Add block output selection to graph CLI and MCP search/context presentation while keeping JSON as the default structured payload. Update generated and root agent instructions so agent exploration uses block format instead of JSON. Constraint: JSON must remain available for strict structured APIs and tests Rejected: Replacing JSON output globally | graph schema/query/API consumers still need strict structured payloads Rejected: Creating root CLAUDE.md | user requested generator support without adding the file now Confidence: high Scope-risk: moderate Directive: Treat block output as presentation-only; retrieval payloads and MCP structuredContent should stay JSON Tested: ./.venv/bin/ruff check .; ./.venv/bin/pytest -q; ./.venv/bin/codebase-graph graph-search SearchService --repo-root . --no-refresh --detail slim --context-limit 1 --format block; ./.venv/bin/codebase-graph setup --repo-root . --mcp-client none Not-tested: External MCP clients beyond the stdio protocol test * chore: target file access restriction * refactor(cli): prevent graph command contract drift Centralize graph CLI and MCP tool metadata behind shared command specs, and move MCP installer client behavior into a single strategy registry. Keep legacy search/context commands and runtime block output compatible while reducing duplicate payload and serialization shaping. Constraint: Public CLI flags and JSON/block output shapes must remain compatible Rejected: Remove legacy search/context commands | existing tests and callers still cover them as supported compatibility surfaces Rejected: Delete parseable block format | retained as debug/test support while runtime keeps compact agent blocks Confidence: high Scope-risk: moderate Directive: Add future graph tools or MCP install clients through the shared registries rather than branching in CLI/MCP dispatch Tested: ./.venv/bin/pytest -q Tested: ./.venv/bin/ruff check Tested: ./.venv/bin/codebase-graph setup --repo-root . --mcp-client none Not-tested: Static type checking; no mypy or pyright executable is installed in the project venv * fix(setup): align generated agent instructions The generated codebaseGraph instruction block now matches the current AGENTS.md wording and the refactored CLI guidance. The stale extra prohibition sentence caused setup regeneration churn without reflecting a CLI requirement. Constraint: Generated instructions must stay stable when codebase-graph setup is rerun after CLI refactors Rejected: Update AGENTS.md directly | regeneration already reports the current file as unchanged once the template is fixed Confidence: high Scope-risk: narrow Directive: Keep instruction text limited to durable CLI workflow guidance, not extra behavioral rules outside the CLI contract Tested: ./.venv/bin/pytest -q tests/test_setup_workflow.py Tested: ./.venv/bin/ruff check src/codebase_graph/setup/instructions.py tests/test_setup_workflow.py Tested: ./.venv/bin/codebase-graph setup --repo-root . --mcp-client none Not-tested: Full test suite; change is limited to setup instruction text and its regression assertion * fix(setup): preserve graph-first instruction Restore the user-authored graph-first reading rule in both the generated instruction block and the checked-in AGENTS.md block, using the requested plural target-files wording. Constraint: Generated AGENTS.md guidance must preserve user-authored workflow policy, not just CLI flag guidance Rejected: Keep the previous generator cleanup | it removed an intentional instruction from the generated block Confidence: high Scope-risk: narrow Directive: Do not remove the graph-first source-reading sentence from setup instructions without explicit user approval Tested: ./.venv/bin/pytest -q tests/test_setup_workflow.py Tested: ./.venv/bin/ruff check src/codebase_graph/setup/instructions.py tests/test_setup_workflow.py Not-tested: Full test suite; change is limited to generated instruction text and its setup regression assertion * docs: reduce README duplication for clearer onboarding The README had grown into a mixed quick start, MCP reference, release note, and troubleshooting index. This keeps the public landing page focused on installation, MCP use, CLI workflow, development, and links to deeper release/security docs. Constraint: README should align with current setup and standalone MCP install naming behavior Rejected: Keep full client config examples in README | dry-run output is the source of truth and avoids duplicating adapter-specific formats Confidence: high Scope-risk: narrow Directive: Keep README focused on user onboarding; put detailed release and client-format procedures in dedicated docs or generated dry-run output Tested: ./.venv/bin/python -m pytest; ./.venv/bin/ruff check .; codebase-graph setup --repo-root . --mcp-client none --instructions-target skip --- AGENTS.md | 7 +- README.md | 214 +++------- src/codebase_graph/cli/__init__.py | 394 +++++++------------ src/codebase_graph/ingest/materializer.py | 15 + src/codebase_graph/mcp/graph_commands.py | 281 +++++++++++++ src/codebase_graph/mcp/tools.py | 109 ++--- src/codebase_graph/retrieval/__init__.py | 6 + src/codebase_graph/retrieval/block_format.py | 52 ++- src/codebase_graph/setup/installer.py | 171 +++++--- src/codebase_graph/setup/instructions.py | 7 +- src/codebase_graph/setup/orchestrator.py | 32 +- tests/test_graph_output_block_format.py | 29 +- tests/test_mcp_installer.py | 14 + tests/test_mcp_portability.py | 10 + tests/test_search.py | 129 +++++- tests/test_setup_workflow.py | 31 +- 16 files changed, 922 insertions(+), 579 deletions(-) create mode 100644 src/codebase_graph/mcp/graph_commands.py diff --git a/AGENTS.md b/AGENTS.md index ba6eb95..fc9f9b8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,8 +1,9 @@ ## codebaseGraph workflow -- Treat the repo-local `.codebaseGraph` graph as the project operating source of truth. -- Use `/Users/rabii/Projects/Repositories/codebaseGraph/.venv/bin/codebase-graph graph-search --repo-root . --no-refresh --detail slim --context-limit 1 --json` before answering repo-structure questions or performing coding tasks. -- Use `/Users/rabii/Projects/Repositories/codebaseGraph/.venv/bin/codebase-graph graph-context --repo-root . --profile --no-refresh --detail slim --context-limit 2 --json` when relationships or nearby evidence matter; useful profiles include `definitions`, `dependencies`, `callgraph`, `docs`, `runtime`, and `change_impact`. +- Treat the repo-local `.codebaseGraph` graph as the project operating source of truth. It is prohibited to read the code source before you find the target files using the graph. +- AI agents must use block format for `graph-search` and `graph-context`; reserve `--json` for tests, APIs, or explicit structured-payload debugging. +- Use `/Users/rabii/Projects/Repositories/codebaseGraph/.venv/bin/codebase-graph graph-search --repo-root . --no-refresh --detail slim --context-limit 1 --format block` before answering repo-structure questions or performing coding tasks. +- Use `/Users/rabii/Projects/Repositories/codebaseGraph/.venv/bin/codebase-graph graph-context --repo-root . --profile --no-refresh --detail slim --context-limit 2 --format block` when relationships or nearby evidence matter; useful profiles include `definitions`, `dependencies`, `callgraph`, `docs`, `runtime`, and `change_impact`. - For architecture orientation, run `/Users/rabii/Projects/Repositories/codebaseGraph/.venv/bin/codebase-graph graph-architecture-queries`, then execute selected read-only statements with `/Users/rabii/Projects/Repositories/codebaseGraph/.venv/bin/codebase-graph graph-query "" --repo-root .`. - Use `/Users/rabii/Projects/Repositories/codebaseGraph/.venv/bin/codebase-graph graph-schema` or `/Users/rabii/Projects/Repositories/codebaseGraph/.venv/bin/codebase-graph graph-query-helpers` before writing raw graph queries, add `--pretty` for indented JSON when humans need to inspect output, and keep `graph-query` read-only. - Refresh the graph with `/Users/rabii/Projects/Repositories/codebaseGraph/.venv/bin/codebase-graph setup --repo-root . --mcp-client none` when files change materially. Setup config: `/Users/rabii/Projects/Repositories/codebaseGraph/.codebaseGraph/config.json`. diff --git a/README.md b/README.md index 1cf97c8..400d9dd 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,17 @@ # codebaseGraph -`codebaseGraph` is a generic project/code knowledge graph engine for coding repositories. It materializes Python source, `AGENTS.md`, `CLAUDE.md`, Markdown, and MDX files into a LadyBugDB-backed graph, then exposes search, compact context, schema, query helpers, and a read-only MCP tool surface for coding agents. +`codebaseGraph` builds a repo-local knowledge graph for coding agents. It materializes Python source, `AGENTS.md`, +`CLAUDE.md`, Markdown, and MDX files into a LadyBugDB-backed graph, then exposes search, compact context, schema, query +helpers, and read-only MCP tools. -LadyBugDB is a required runtime dependency. A normal production install must include `real_ladybug`; setup fails before creating repository state if the runtime cannot open a graph database. +Requires Python 3.10+ and a package build that includes `real_ladybug`. -## Production install +## Quick start ```bash python -m pip install cbasegraph -``` - -From a repository root, run: - -```bash codebase-graph setup --repo-root . +codebase-graph graph-search SampleService --repo-root . --no-refresh --format block ``` Setup creates: @@ -25,138 +23,66 @@ Setup creates: _graph.ldb ``` -For a repository named `my-service`, the database path is exactly `.codebaseGraph/my-service_graph.ldb`. +For a repository named `my-service`, the database path is `.codebaseGraph/my-service_graph.ldb`. -The setup command also: +The setup command materializes the graph, writes or updates one marked codebaseGraph block in `AGENTS.md` or +`CLAUDE.md`, and installs a Codex MCP client entry unless skipped. -- Materializes the repository graph into the repo-local database. -- Writes or updates one marked codebaseGraph block in `AGENTS.md` or `CLAUDE.md`. -- Installs an MCP client entry named `codebase_graph`, unless skipped. - -Useful options: +Useful setup options: ```bash codebase-graph setup --repo-root /path/to/repo -codebase-graph setup --mcp-client codex codebase-graph setup --mcp-client claude -codebase-graph setup --mcp-client claude-project codebase-graph setup --mcp-client lmstudio -codebase-graph setup --mcp-client hermes -codebase-graph setup --mcp-client openclaw -codebase-graph setup --mcp-client generic -codebase-graph setup --mcp-config-path /tmp/client-config -codebase-graph setup --dry-run codebase-graph setup --skip-mcp-config codebase-graph setup --instructions-target claude +codebase-graph setup --dry-run --pretty ``` -`--dry-run` returns the raw server descriptor plus the exact client patch or payload without writing repository graph state, instruction files, or MCP client files. - -## MCP installation - -The user-facing installer is: +## MCP install ```bash -codebase-graph mcp install +codebase-graph mcp install --client codex ``` -By default this installs Codex with a repository-specific server name, for example `codebase_graph_my_service`. It builds the server descriptor from `.codebaseGraph/config.json`, uses the supported native client CLI when available, and falls back to the adapter file writer when the CLI is missing or fails. +Supported clients are `codex`, `claude`, `claude-project`, `lmstudio`, `hermes`, `openclaw`, and `generic`. + +Server naming: + +- `codebase-graph setup` installs the default MCP server as `codebase_graph`. +- Standalone `codebase-graph mcp install` defaults to `codebase_graph_`. +- Use `--name codebase_graph` to override the standalone installer name. -Useful installer options: +The installer builds the server descriptor from `.codebaseGraph/config.json`, uses a supported native client CLI when +available, and falls back to writing the client config file directly. Use `--dry-run --json` to inspect the emitted +command or config patch before writing, and `--verify` to run a stdio smoke test after installation. ```bash -codebase-graph mcp install --client codex codebase-graph mcp install --client claude --scope user codebase-graph mcp install --client claude-project -codebase-graph mcp install --client lmstudio -codebase-graph mcp install --client hermes -codebase-graph mcp install --client openclaw -codebase-graph mcp install --client generic codebase-graph mcp install --client all --dry-run --json -codebase-graph mcp install --name codebase_graph codebase-graph mcp install --config-path /path/to/.codebaseGraph/config.json codebase-graph mcp install --verify ``` -Native CLI installers are attempted first for Codex, Claude, Claude project scope, and OpenClaw: - -```bash -codex mcp add -- -claude mcp add --transport stdio --scope -- -openclaw mcp set '' -``` - -If native installation is unavailable, codebaseGraph writes the client config file directly. `setup --mcp-client ...` remains supported and delegates to the same installer behavior after materializing graph state and updating instructions. The default MCP server name is `codebase_graph`, which avoids mixed-case tool namespace issues in clients that normalize or validate MCP labels strictly. - -`--dry-run` reports the native command or emitted file patch without calling native CLIs or writing files. `--verify` runs a direct stdio MCP smoke test and, where available, asks the client CLI whether it can see the server. - ## MCP usage -Setup and install build one canonical server descriptor and serialize it into the selected client format. When run from a virtual environment, the command may be the absolute path to that environment's `codebase-graph` executable so the MCP client can launch it without relying on shell `PATH`. - -Codex uses `~/.codex/config.toml`: - -```toml -[mcp_servers.codebase_graph] -command = "codebase-graph" -args = ["mcp", "serve", "--config", ".codebaseGraph/config.json"] -startup_timeout_sec = 60 -``` - -Claude Desktop, Claude project config, LM Studio, and generic MCP JSON use an `mcpServers` shape: - -```json -{ - "mcpServers": { - "codebase_graph": { - "type": "stdio", - "command": "codebase-graph", - "args": ["mcp", "serve", "--config", ".codebaseGraph/config.json"] - } - } -} -``` - -OpenClaw uses JSON5-compatible JSON under `mcp.servers`, and Hermes emits YAML under `mcp_servers` in `~/.hermes/config.yaml`. LM Studio reads `~/.lmstudio/mcp.json` and requires enabling "Allow calling servers from mcp.json" in the app. Use `codebase-graph mcp install --dry-run --client --json` to inspect the exact emitted command or patch before installation. - -Client examples: - -```bash -codebase-graph mcp install --client codex -codebase-graph mcp install --client claude -codebase-graph mcp install --client claude-project -codebase-graph mcp install --client lmstudio -codebase-graph mcp install --client hermes -codebase-graph mcp install --client openclaw -codebase-graph mcp install --client generic --dry-run --json -``` - -The server can also be run directly: +Stdio is the default transport for local MCP clients: ```bash codebase-graph mcp serve --config .codebaseGraph/config.json codebase-graph-mcp --config .codebaseGraph/config.json ``` -Stdio is the default transport for local MCP clients. An optional local Streamable HTTP transport is available for clients that connect to an HTTP endpoint: +HTTP is available for local endpoint clients: ```bash codebase-graph mcp http --config .codebaseGraph/config.json --host 127.0.0.1 --port 8765 ``` -The HTTP transport rejects non-local bind hosts unless `--allow-remote` is passed. Keep it bound to `127.0.0.1` -for normal use. Remote binding requires a bearer token: - -```bash -CODEBASE_GRAPH_MCP_TOKEN="$(openssl rand -hex 32)" -codebase-graph mcp http --config .codebaseGraph/config.json --host 0.0.0.0 --allow-remote --auth-token-env CODEBASE_GRAPH_MCP_TOKEN -``` - -Clients must send `Authorization: Bearer `. The token gate does not add TLS, rate limiting, authorization scopes, or -a multi-user session model; put remote HTTP behind a trusted network boundary and TLS-terminating proxy. - -HTTP clients must start with JSON-RPC `initialize`, then send the returned `Mcp-Session-Id` response header on later -requests. Requests without a known session id are rejected before tool dispatch. +Keep HTTP bound to `127.0.0.1` for normal use. Remote binding requires `--allow-remote` and a bearer token, but does not +provide TLS, rate limiting, authorization scopes, or a multi-user security model. HTTP clients must initialize first and +send the returned `Mcp-Session-Id` header on later requests. Available MCP tools: @@ -168,83 +94,53 @@ Available MCP tools: - `graph_architecture_queries` - `graph_query` with write-like statements blocked -`graph_query` returns at most 1,000 rows per call and fetches only one extra row to determine whether the result was -truncated. Add a narrower `MATCH` pattern or a query-side `LIMIT` for broader graph exploration. - -For coding-task architecture orientation, call `graph_architecture_queries` first to fetch the grouped read-only Cypher catalog, then run selected statements with `graph_query`. - -## Operational diagnostics - -Runtime warning and error paths emit structured JSON events to stderr. Set `CODEBASE_GRAPH_LOG_LEVEL=INFO` to include -setup start/completion diagnostics; the default level is `WARNING`. +`graph_query` returns at most 1,000 rows per call. Add a narrower `MATCH` pattern or a query-side `LIMIT` for broader +graph exploration. -Examples of emitted events include: +## CLI workflow -- `setup.failed` -- `mcp.tool_error` -- `mcp.stdio_parse_error` -- `mcp.http_forbidden_origin` -- `materializer.lock_exists` -- `materializer.stale_lock_removed` - -## CLI graph workflow - -The CLI exposes the same graph workflow as the MCP tools, which is useful in clients that do not surface MCP tools directly: +The CLI mirrors the MCP tools for clients that do not surface MCP directly: ```bash codebase-graph graph-health --repo-root . -codebase-graph graph-search SampleService --repo-root . --no-refresh --detail slim --context-limit 1 --json -codebase-graph graph-context SampleService --repo-root . --profile definitions --no-refresh --detail slim --context-limit 2 --json -codebase-graph graph-schema -codebase-graph graph-query-helpers -codebase-graph graph-architecture-queries --group overview +codebase-graph graph-context SampleService --repo-root . --profile definitions --format block codebase-graph graph-query "MATCH (n) RETURN count(n) AS total_nodes LIMIT 1" --repo-root . ``` -CLI JSON output is minified by default to reduce tokens. Add `--pretty` to JSON-producing commands when you want indented output. Retrieval commands support `--detail standard|slim`; `standard` keeps the full payload, while `slim` drops score diagnostics and duplicate or empty summary fields. - -`graph-query` blocks write-like statements and should be used read-only. The older `search` and `context` commands remain available. Setup reports the explicit database and manifest paths to use with them when needed: +Use `--format block` for agent-facing output and `--json --pretty` for structured inspection. Retrieval commands also +support `--detail standard|slim`; `slim` drops score diagnostics and duplicate or empty summary fields. -```bash -codebase-graph search SampleService \ - --source-root . \ - --db .codebaseGraph/_graph.ldb \ - --manifest .codebaseGraph/manifest.json -``` +For coding-task architecture orientation, call `graph_architecture_queries` first, then run selected statements with +`graph_query`. -## Development install +## Development ```bash python -m pip install -e .[dev] -``` - -Run checks: - -```bash python -m pytest ruff check . ``` -## CI and releases - -GitHub Actions runs pytest across Linux, macOS, and Windows for Python 3.10 through 3.14, plus ruff, supply-chain, and package-build validation. Supply-chain checks include dependency consistency, vulnerability advisory scanning, Dependabot update coverage, immutable GitHub Action pins, and CycloneDX SBOM generation. Built wheels and source distributions are smoke-tested with `setup`, `graph-health`, `graph-search`, and a stdio MCP handshake before release. Releases are managed by release-please, use tag-derived package versions, create GitHub Releases with distribution assets and SBOMs, and publish to PyPI through Trusted Publishing. - -Run `python scripts/check_release_gate.py` for local release-gate checks. Use the `--production` confirmations documented in [docs/release.md](docs/release.md) before publishing. - -Conda distribution uses the conda-forge staged-recipes path rather than direct Anaconda.org uploads. See [docs/release.md](docs/release.md) for the release workflow and conda-forge submission checklist. +## Release and security -## Security +CI runs pytest across Linux, macOS, and Windows for Python 3.10 through 3.14, plus ruff, package-build checks, +supply-chain validation, and smoke tests. See [docs/release.md](docs/release.md) for the full release process and +conda-forge checklist. -Report suspected vulnerabilities privately. See [SECURITY.md](SECURITY.md) for supported versions, reporting expectations, and the local-first MCP security boundary. +Report suspected vulnerabilities privately. See [SECURITY.md](SECURITY.md) for supported versions, reporting +expectations, and the local-first MCP security boundary. ## Troubleshooting -- Missing LadyBugDB: install a package build that includes `real_ladybug`; setup will fail before creating `.codebaseGraph`. +- Missing LadyBugDB: install a package build that includes `real_ladybug`; setup fails before creating `.codebaseGraph` + if the runtime cannot open a graph database. - Stale graph: rerun `codebase-graph setup --repo-root .` after material source or documentation changes. -- Broken Codex config: rerun `codebase-graph mcp install --client codex --verify`, then check `codex mcp list`. -- Broken Claude config: rerun `codebase-graph mcp install --client claude --scope user --verify` or `codebase-graph mcp install --client claude-project --verify`. -- Broken LM Studio, Hermes, OpenClaw, or generic config: run `codebase-graph mcp install --client --dry-run --json` first, then inspect the emitted payload and target path. -- PATH or executable issues: run setup from the virtual environment that contains `codebase-graph`; the descriptor prefers that absolute executable path. -- Direct smoke test: run `codebase-graph mcp serve --config .codebaseGraph/config.json` and send MCP `initialize`, `tools/list`, and `tools/call` JSON-RPC messages over stdio. -- Unsupported files: binary, vendor, cache, virtualenv, build, dist, `.codebase_graph`, and `.codebaseGraph` paths are skipped. -- Lock/contention errors: stop other graph materialization or setup processes using the same `.codebaseGraph/_graph.ldb`. Stale locks with dead writer PIDs are removed automatically; if the error remains, inspect the `.ldb.lock` file before removing it manually. +- Broken client config: rerun `codebase-graph mcp install --client --verify`. +- PATH or executable issues: run setup from the virtual environment that contains `codebase-graph`; the descriptor + prefers that absolute executable path. +- Unsupported files: binary, vendor, cache, virtualenv, build, dist, `.codebase_graph`, and `.codebaseGraph` paths are + skipped. +- Lock errors: stop other graph materialization or setup processes using the same + `.codebaseGraph/_graph.ldb`. Stale locks with dead writer PIDs are removed automatically; if the error + remains, inspect the `.ldb.lock` file before removing it manually. +- Diagnostics: set `CODEBASE_GRAPH_LOG_LEVEL=INFO` to include setup start/completion events on stderr. diff --git a/src/codebase_graph/cli/__init__.py b/src/codebase_graph/cli/__init__.py index 2b8af75..b86f24b 100644 --- a/src/codebase_graph/cli/__init__.py +++ b/src/codebase_graph/cli/__init__.py @@ -8,126 +8,23 @@ from codebase_graph.db import create_ladybug_database from codebase_graph.ingest import GraphMaterializer +from codebase_graph.mcp.graph_commands import ( + add_compact_context_arguments, + add_json_output_arguments, + graph_command_names, + graph_command_spec, + graph_command_specs, +) from codebase_graph.mcp.runtime import runtime_config from codebase_graph.mcp.tools import handle_tool_call -from codebase_graph.ontology import CONTEXT_PROFILES, QUERY_HELPERS, schema_payload -from codebase_graph.reasoning import architecture_query_catalog -from codebase_graph.retrieval import SearchRequest, SearchService +from codebase_graph.retrieval import SearchRequest, SearchService, serialize_graph_block from codebase_graph.setup import SetupError, SetupOptions, run_setup from codebase_graph.setup.clients import supported_client_ids from codebase_graph.setup.installer import McpInstallOptions, install_mcp_clients, supported_install_client_ids def main(argv: Sequence[str] | None = None) -> int: - parser = argparse.ArgumentParser(prog="codebase-graph") - subparsers = parser.add_subparsers(dest="command", required=True) - - materialize_parser = subparsers.add_parser("materialize", help="Materialize the code graph") - materialize_parser.add_argument("--source-root", default=".", help="Repository or source root to scan") - materialize_parser.add_argument("--db", default=None, help="LadybugDB path; defaults under .codebaseGraph") - materialize_parser.add_argument("--manifest", default=None, help="Manifest path; defaults under .codebaseGraph") - materialize_parser.add_argument("--mode", choices=("full", "changed"), default="changed") - materialize_parser.add_argument("--no-fts", action="store_true", help="Skip FTS index creation") - _add_json_output_arguments(materialize_parser) - - search_parser = subparsers.add_parser("search", help="Search the code graph with compact context") - _add_search_arguments(search_parser) - - context_parser = subparsers.add_parser("context", help="Return compact context for a search query") - _add_search_arguments(context_parser) - - graph_health_parser = subparsers.add_parser("graph-health", help="Check configured graph paths") - _add_runtime_arguments(graph_health_parser) - _add_json_output_arguments(graph_health_parser) - - graph_search_parser = subparsers.add_parser("graph-search", help="Search the code graph with compact context") - graph_search_parser.add_argument("query", help="Search query") - _add_compact_context_arguments(graph_search_parser) - _add_runtime_arguments(graph_search_parser) - _add_graph_compatibility_arguments(graph_search_parser) - - graph_context_parser = subparsers.add_parser("graph-context", help="Return compact graph context") - graph_context_parser.add_argument("query", nargs="?", help="Search query") - graph_context_parser.add_argument("--node-id", default=None, help="Explicit graph node id") - graph_context_parser.add_argument("--node-type", default=None, help="Explicit graph node type") - _add_compact_context_arguments(graph_context_parser) - _add_runtime_arguments(graph_context_parser) - _add_graph_compatibility_arguments(graph_context_parser) - - graph_schema_parser = subparsers.add_parser("graph-schema", help="Return ontology schema, indexes, profiles, and helpers") - _add_json_output_arguments(graph_schema_parser) - graph_query_helpers_parser = subparsers.add_parser("graph-query-helpers", help="Return named read-only graph query helpers") - _add_json_output_arguments(graph_query_helpers_parser) - - graph_architecture_parser = subparsers.add_parser( - "graph-architecture-queries", - help="Return the architecture-discovery query catalog", - ) - graph_architecture_parser.add_argument("--group", default=None, help="Optional architecture query group") - _add_json_output_arguments(graph_architecture_parser) - - graph_query_parser = subparsers.add_parser("graph-query", help="Execute a restricted read-only graph query") - graph_query_parser.add_argument("statement", help="Read-only graph query statement") - graph_query_parser.add_argument("--parameters", default="{}", help="JSON object with query parameters") - graph_query_parser.add_argument("--limit", type=int, default=100, help="Maximum rows to return") - _add_runtime_arguments(graph_query_parser) - _add_json_output_arguments(graph_query_parser) - - setup_parser = subparsers.add_parser("setup", help="Bootstrap codebaseGraph state for a repository") - setup_parser.add_argument("--repo-root", default=".", help="Repository root to configure") - setup_parser.add_argument("--mcp-client", choices=supported_client_ids(), default="codex") - setup_parser.add_argument("--mcp-config-path", default=None, help="Override MCP client config path") - setup_parser.add_argument("--skip-mcp-config", action="store_true", help="Do not write MCP client config") - setup_parser.add_argument("--dry-run", action="store_true", help="Return the MCP config patch without writing it") - setup_parser.add_argument( - "--instructions-target", - choices=("auto", "agents", "claude", "skip"), - default="auto", - help="Instruction file to update", - ) - setup_parser.add_argument("--mode", choices=("full", "changed"), default="changed", help="Materialization mode") - setup_parser.add_argument("--json", action="store_true", help="Emit JSON output") - _add_json_output_arguments(setup_parser) - - mcp_parser = subparsers.add_parser("mcp", help="Run or inspect the MCP server") - mcp_subparsers = mcp_parser.add_subparsers(dest="mcp_command", required=True) - install_parser = mcp_subparsers.add_parser("install", help="Install the MCP server in a supported client") - install_parser.add_argument("--client", choices=supported_install_client_ids(include_all=True), default="codex") - install_parser.add_argument("--scope", choices=("local", "user", "project"), default="local") - install_parser.add_argument("--name", default=None, help="MCP server name; defaults to codebase_graph-") - install_parser.add_argument("--config-path", default=None, help="Path to .codebaseGraph/config.json") - install_parser.add_argument("--client-config-path", default=None, help="Override the target MCP client config path") - install_parser.add_argument("--repo-root", default=".", help="Repository root used to find .codebaseGraph/config.json") - install_parser.add_argument("--dry-run", action="store_true", help="Show the install action without writing or invoking CLIs") - install_parser.add_argument("--verify", action="store_true", help="Run direct MCP smoke checks after installation") - install_parser.add_argument("--json", action="store_true", help="Emit JSON output") - _add_json_output_arguments(install_parser) - - serve_parser = mcp_subparsers.add_parser("serve", help="Serve graph tools over MCP stdio") - serve_parser.add_argument("--repo-root", default=".", help="Repository root containing .codebaseGraph/config.json") - serve_parser.add_argument("--config", default=None, help="Path to .codebaseGraph/config.json") - serve_parser.add_argument("--db", default=None, help="Override LadyBugDB path") - serve_parser.add_argument("--manifest", default=None, help="Override manifest path") - http_parser = mcp_subparsers.add_parser("http", help="Serve graph tools over Streamable HTTP") - http_parser.add_argument("--repo-root", default=".", help="Repository root containing .codebaseGraph/config.json") - http_parser.add_argument("--config", default=None, help="Path to .codebaseGraph/config.json") - http_parser.add_argument("--db", default=None, help="Override LadyBugDB path") - http_parser.add_argument("--manifest", default=None, help="Override manifest path") - http_parser.add_argument("--host", default="127.0.0.1", help="HTTP bind host; default keeps the server local") - http_parser.add_argument("--port", type=int, default=8765, help="HTTP bind port") - http_parser.add_argument("--path", default="/mcp", help="MCP HTTP endpoint path") - http_parser.add_argument( - "--allow-remote", - action="store_true", - help="Allow binding MCP HTTP to a non-local host; requires an auth token", - ) - http_parser.add_argument( - "--auth-token", - default=None, - help="Bearer token required for HTTP requests; prefer --auth-token-env to avoid shell history exposure", - ) - http_parser.add_argument("--auth-token-env", default=None, help="Environment variable containing the HTTP bearer token") - + parser = _build_parser() args = parser.parse_args(argv) if args.command == "materialize": materializer = GraphMaterializer( @@ -140,79 +37,12 @@ def main(argv: Sequence[str] | None = None) -> int: result = materializer.materialize(mode=args.mode) finally: materializer.close() - _print_json(_result_payload(result), args) + _print_json(result.as_dict(), args) return 0 if args.command in {"search", "context"}: - request = SearchRequest( - query=args.query, - limit=args.limit, - profile=args.profile, - budget=args.budget, - max_depth=args.max_depth, - context_limit=args.context_limit, - detail=args.detail, - ) - try: - request.validate() - except ValueError as exc: - parser.error(str(exc)) - materializer = GraphMaterializer( - Path(args.source_root), - db_path=args.db, - manifest_path=args.manifest, - include_fts=True, - ) - if args.no_refresh: - with create_ladybug_database(materializer.db_path, include_fts=True, read_only=True) as store: - payload = SearchService(store).search(request) - else: - try: - materializer.materialize(mode="changed") - payload = SearchService(materializer.store).search(request) - finally: - materializer.close() - _print_json(payload.as_dict(detail=args.detail), args) - return 0 - if args.command == "graph-health": - return _print_tool_payload(parser, "graph_health", {}, args) - if args.command == "graph-search": - return _print_tool_payload(parser, "graph_search", _search_arguments_payload(args), args) - if args.command == "graph-context": - if not args.query and not (args.node_id and args.node_type): - parser.error("graph-context requires a query or both --node-id and --node-type") - if (args.node_id and not args.node_type) or (args.node_type and not args.node_id): - parser.error("graph-context explicit lookup requires both --node-id and --node-type") - payload = _search_arguments_payload(args) - if args.node_id and args.node_type: - payload["node_id"] = args.node_id - payload["node_type"] = args.node_type - return _print_tool_payload(parser, "graph_context", payload, args) - if args.command == "graph-schema": - _print_json(schema_payload(), args) - return 0 - if args.command == "graph-query-helpers": - _print_json({"query_helpers": [helper.as_dict() for helper in QUERY_HELPERS]}, args) - return 0 - if args.command == "graph-architecture-queries": - try: - payload = architecture_query_catalog(group=args.group) - except ValueError as exc: - parser.error(str(exc)) - _print_json(payload, args) - return 0 - if args.command == "graph-query": - try: - parameters = json.loads(args.parameters) - except json.JSONDecodeError as exc: - parser.error(f"graph-query --parameters must be a JSON object: {exc}") - if not isinstance(parameters, dict): - parser.error("graph-query --parameters must be a JSON object") - return _print_tool_payload( - parser, - "graph_query", - {"statement": args.statement, "parameters": parameters, "limit": args.limit}, - args, - ) + return _run_legacy_search_command(parser, args) + if args.command in graph_command_names(): + return _run_graph_command(parser, args) if args.command == "setup": try: result = run_setup( @@ -285,38 +115,95 @@ def main(argv: Sequence[str] | None = None) -> int: return 2 +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="codebase-graph") + subparsers = parser.add_subparsers(dest="command", required=True) + + materialize_parser = subparsers.add_parser("materialize", help="Materialize the code graph") + materialize_parser.add_argument("--source-root", default=".", help="Repository or source root to scan") + materialize_parser.add_argument("--db", default=None, help="LadybugDB path; defaults under .codebaseGraph") + materialize_parser.add_argument("--manifest", default=None, help="Manifest path; defaults under .codebaseGraph") + materialize_parser.add_argument("--mode", choices=("full", "changed"), default="changed") + materialize_parser.add_argument("--no-fts", action="store_true", help="Skip FTS index creation") + _add_json_output_arguments(materialize_parser) + + search_parser = subparsers.add_parser("search", help="Search the code graph with compact context") + _add_search_arguments(search_parser) + + context_parser = subparsers.add_parser("context", help="Return compact context for a search query") + _add_search_arguments(context_parser) + + for spec in graph_command_specs(): + graph_parser = subparsers.add_parser(spec.command_name, help=spec.help) + spec.add_arguments(graph_parser) + + setup_parser = subparsers.add_parser("setup", help="Bootstrap codebaseGraph state for a repository") + setup_parser.add_argument("--repo-root", default=".", help="Repository root to configure") + setup_parser.add_argument("--mcp-client", choices=supported_client_ids(), default="codex") + setup_parser.add_argument("--mcp-config-path", default=None, help="Override MCP client config path") + setup_parser.add_argument("--skip-mcp-config", action="store_true", help="Do not write MCP client config") + setup_parser.add_argument("--dry-run", action="store_true", help="Return the MCP config patch without writing it") + setup_parser.add_argument( + "--instructions-target", + choices=("auto", "agents", "claude", "skip"), + default="auto", + help="Instruction file to update", + ) + setup_parser.add_argument("--mode", choices=("full", "changed"), default="changed", help="Materialization mode") + setup_parser.add_argument("--json", action="store_true", help="Emit JSON output") + _add_json_output_arguments(setup_parser) + + mcp_parser = subparsers.add_parser("mcp", help="Run or inspect the MCP server") + mcp_subparsers = mcp_parser.add_subparsers(dest="mcp_command", required=True) + install_parser = mcp_subparsers.add_parser("install", help="Install the MCP server in a supported client") + install_parser.add_argument("--client", choices=supported_install_client_ids(include_all=True), default="codex") + install_parser.add_argument("--scope", choices=("local", "user", "project"), default="local") + install_parser.add_argument("--name", default=None, help="MCP server name; defaults to codebase_graph-") + install_parser.add_argument("--config-path", default=None, help="Path to .codebaseGraph/config.json") + install_parser.add_argument("--client-config-path", default=None, help="Override the target MCP client config path") + install_parser.add_argument("--repo-root", default=".", help="Repository root used to find .codebaseGraph/config.json") + install_parser.add_argument("--dry-run", action="store_true", help="Show the install action without writing or invoking CLIs") + install_parser.add_argument("--verify", action="store_true", help="Run direct MCP smoke checks after installation") + install_parser.add_argument("--json", action="store_true", help="Emit JSON output") + _add_json_output_arguments(install_parser) + + serve_parser = mcp_subparsers.add_parser("serve", help="Serve graph tools over MCP stdio") + serve_parser.add_argument("--repo-root", default=".", help="Repository root containing .codebaseGraph/config.json") + serve_parser.add_argument("--config", default=None, help="Path to .codebaseGraph/config.json") + serve_parser.add_argument("--db", default=None, help="Override LadyBugDB path") + serve_parser.add_argument("--manifest", default=None, help="Override manifest path") + http_parser = mcp_subparsers.add_parser("http", help="Serve graph tools over Streamable HTTP") + http_parser.add_argument("--repo-root", default=".", help="Repository root containing .codebaseGraph/config.json") + http_parser.add_argument("--config", default=None, help="Path to .codebaseGraph/config.json") + http_parser.add_argument("--db", default=None, help="Override LadyBugDB path") + http_parser.add_argument("--manifest", default=None, help="Override manifest path") + http_parser.add_argument("--host", default="127.0.0.1", help="HTTP bind host; default keeps the server local") + http_parser.add_argument("--port", type=int, default=8765, help="HTTP bind port") + http_parser.add_argument("--path", default="/mcp", help="MCP HTTP endpoint path") + http_parser.add_argument( + "--allow-remote", + action="store_true", + help="Allow binding MCP HTTP to a non-local host; requires an auth token", + ) + http_parser.add_argument( + "--auth-token", + default=None, + help="Bearer token required for HTTP requests; prefer --auth-token-env to avoid shell history exposure", + ) + http_parser.add_argument("--auth-token-env", default=None, help="Environment variable containing the HTTP bearer token") + return parser + + def _add_search_arguments(parser: argparse.ArgumentParser) -> None: parser.add_argument("query", help="Search query") parser.add_argument("--source-root", default=".", help="Repository or source root to search") parser.add_argument("--db", default=None, help="LadybugDB path; defaults under .codebaseGraph") parser.add_argument("--manifest", default=None, help="Manifest path; defaults under .codebaseGraph") - _add_compact_context_arguments(parser) + add_compact_context_arguments(parser) parser.add_argument("--no-refresh", action="store_true", help="Query the existing graph without changed materialization") parser.add_argument("--json", action="store_true", help="Emit compact JSON output") -def _add_compact_context_arguments(parser: argparse.ArgumentParser) -> None: - parser.add_argument("--limit", type=int, default=3, help="Maximum search hits to return") - parser.add_argument("--profile", choices=sorted(CONTEXT_PROFILES), default="brief", help="Context profile") - parser.add_argument("--budget", type=int, default=600, help="Approximate per-hit context character budget") - parser.add_argument("--max-depth", type=int, default=None, help="Override the context profile depth") - parser.add_argument("--context-limit", type=int, default=3, help="Maximum context items per search hit") - parser.add_argument("--detail", choices=("standard", "slim"), default="standard", help="Output detail level") - _add_json_output_arguments(parser) - - -def _add_runtime_arguments(parser: argparse.ArgumentParser) -> None: - parser.add_argument("--repo-root", default=".", help="Repository root containing .codebaseGraph/config.json") - parser.add_argument("--config", default=None, help="Path to .codebaseGraph/config.json") - parser.add_argument("--db", default=None, help="Override LadyBugDB path") - parser.add_argument("--manifest", default=None, help="Override manifest path") - - -def _add_graph_compatibility_arguments(parser: argparse.ArgumentParser) -> None: - parser.add_argument("--no-refresh", action="store_true", help="Accepted for search/context command parity") - parser.add_argument("--json", action="store_true", help="Accepted for search/context command parity") - - def _runtime(args: argparse.Namespace) -> object: return runtime_config( repo_root=args.repo_root, @@ -326,65 +213,80 @@ def _runtime(args: argparse.Namespace) -> object: ) -def _search_arguments_payload(args: argparse.Namespace) -> dict[str, object]: - payload: dict[str, object] = { - "limit": args.limit, - "profile": args.profile, - "budget": args.budget, - "context_limit": args.context_limit, - "detail": args.detail, - } - if args.query: - payload["query"] = args.query - if args.max_depth is not None: - payload["max_depth"] = args.max_depth - return payload - - -def _print_tool_payload( - parser: argparse.ArgumentParser, - tool_name: str, - arguments: dict[str, object], - args: argparse.Namespace, -) -> int: +def _run_legacy_search_command(parser: argparse.ArgumentParser, args: argparse.Namespace) -> int: + try: + request = _search_request_from_args(args) + except ValueError as exc: + parser.error(str(exc)) + materializer = GraphMaterializer( + Path(args.source_root), + db_path=args.db, + manifest_path=args.manifest, + include_fts=True, + ) + if args.no_refresh: + with create_ladybug_database(materializer.db_path, include_fts=True, read_only=True) as store: + payload = SearchService(store).search(request) + else: + try: + materializer.materialize(mode="changed") + payload = SearchService(materializer.store).search(request) + finally: + materializer.close() + _print_payload(payload.as_dict(detail=args.detail), args) + return 0 + + +def _search_request_from_args(args: argparse.Namespace) -> SearchRequest: + request = SearchRequest( + query=args.query, + limit=args.limit, + profile=args.profile, + budget=args.budget, + max_depth=args.max_depth, + context_limit=args.context_limit, + detail=args.detail, + ) + request.validate() + return request + + +def _run_graph_command(parser: argparse.ArgumentParser, args: argparse.Namespace) -> int: + spec = graph_command_spec(args.command) try: - payload = handle_tool_call(tool_name, arguments, runtime=_runtime(args)) + arguments = spec.payload_from_args(args) + runtime = _runtime(args) if spec.requires_runtime else None + payload = handle_tool_call(spec.tool_name, arguments, runtime=runtime) except (OSError, ValueError) as exc: parser.error(str(exc)) - _print_json(payload, args) + _print_payload(payload, args) return 0 def _add_json_output_arguments(parser: argparse.ArgumentParser) -> None: - parser.add_argument("--pretty", action="store_true", help="Emit indented JSON output") + add_json_output_arguments(parser) def _print_json(payload: object, args: argparse.Namespace) -> None: print(_json_dumps(payload, pretty=getattr(args, "pretty", False))) +def _print_payload(payload: dict[str, object], args: argparse.Namespace) -> None: + if getattr(args, "json", False): + _print_json(payload, args) + return + if getattr(args, "format", "json") == "block": + print(serialize_graph_block(payload), end="") + return + _print_json(payload, args) + + def _json_dumps(payload: object, *, pretty: bool) -> str: if pretty: return json.dumps(payload, indent=2, sort_keys=True) return json.dumps(payload, separators=(",", ":"), sort_keys=True) -def _result_payload(result: object) -> dict[str, object]: - return { - "mode": getattr(result, "mode"), - "scanned": getattr(result, "scanned"), - "rebuilt": getattr(result, "rebuilt"), - "skipped": getattr(result, "skipped"), - "deleted": getattr(result, "deleted"), - "diagnostics": list(getattr(result, "diagnostics")), - "manifest_path": getattr(result, "manifest_path"), - "rebuilt_paths": list(getattr(result, "rebuilt_paths")), - "skipped_paths": list(getattr(result, "skipped_paths")), - "deleted_paths": list(getattr(result, "deleted_paths")), - "graph_summary": dict(getattr(result, "graph_summary")), - } - - def _print_mcp_install_results(results: Sequence[object]) -> None: for result in results: action = getattr(result, "action") diff --git a/src/codebase_graph/ingest/materializer.py b/src/codebase_graph/ingest/materializer.py index ca3576c..96d7d36 100644 --- a/src/codebase_graph/ingest/materializer.py +++ b/src/codebase_graph/ingest/materializer.py @@ -206,6 +206,21 @@ class MaterializationResult: deleted_paths: tuple[str, ...] graph_summary: Mapping[str, Any] + def as_dict(self) -> dict[str, Any]: + return { + "mode": self.mode, + "scanned": self.scanned, + "rebuilt": self.rebuilt, + "skipped": self.skipped, + "deleted": self.deleted, + "diagnostics": list(self.diagnostics), + "manifest_path": self.manifest_path, + "rebuilt_paths": list(self.rebuilt_paths), + "skipped_paths": list(self.skipped_paths), + "deleted_paths": list(self.deleted_paths), + "graph_summary": dict(self.graph_summary), + } + class GraphMaterializer: def __init__( diff --git a/src/codebase_graph/mcp/graph_commands.py b/src/codebase_graph/mcp/graph_commands.py new file mode 100644 index 0000000..5af427b --- /dev/null +++ b/src/codebase_graph/mcp/graph_commands.py @@ -0,0 +1,281 @@ +from __future__ import annotations + +import argparse +import json +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from typing import Any + +from codebase_graph.ontology import CONTEXT_PROFILES +from codebase_graph.retrieval import DETAIL_LEVELS + + +MAX_GRAPH_QUERY_LIMIT = 1000 + +PayloadBuilder = Callable[[argparse.Namespace], dict[str, Any]] +ArgumentAdder = Callable[[argparse.ArgumentParser], None] + + +@dataclass(frozen=True, slots=True) +class GraphCommandSpec: + command_name: str + tool_name: str + help: str + description: str + input_schema: dict[str, Any] + add_arguments: ArgumentAdder + payload_from_args: PayloadBuilder + requires_runtime: bool = True + + def tool_spec(self) -> dict[str, Any]: + return { + "name": self.tool_name, + "description": self.description, + "inputSchema": self.input_schema, + } + + +def graph_command_specs() -> tuple[GraphCommandSpec, ...]: + return GRAPH_COMMAND_SPECS + + +def graph_command_names() -> set[str]: + return {spec.command_name for spec in GRAPH_COMMAND_SPECS} + + +def graph_tool_specs() -> list[dict[str, Any]]: + return [spec.tool_spec() for spec in GRAPH_COMMAND_SPECS] + + +def graph_command_spec(command_name: str) -> GraphCommandSpec: + for spec in GRAPH_COMMAND_SPECS: + if spec.command_name == command_name: + return spec + raise KeyError(command_name) + + +def search_arguments_payload(args: argparse.Namespace) -> dict[str, Any]: + payload: dict[str, Any] = { + "limit": args.limit, + "profile": args.profile, + "budget": args.budget, + "context_limit": args.context_limit, + "detail": args.detail, + } + if getattr(args, "query", None): + payload["query"] = args.query + if args.max_depth is not None: + payload["max_depth"] = args.max_depth + return payload + + +def _empty_payload(args: argparse.Namespace) -> dict[str, Any]: + return {} + + +def _architecture_payload(args: argparse.Namespace) -> dict[str, Any]: + payload: dict[str, Any] = {} + if args.group: + payload["group"] = args.group + return payload + + +def _context_payload(args: argparse.Namespace) -> dict[str, Any]: + if not args.query and not (args.node_id and args.node_type): + raise ValueError("graph-context requires a query or both --node-id and --node-type") + if (args.node_id and not args.node_type) or (args.node_type and not args.node_id): + raise ValueError("graph-context explicit lookup requires both --node-id and --node-type") + payload = search_arguments_payload(args) + if args.node_id and args.node_type: + payload["node_id"] = args.node_id + payload["node_type"] = args.node_type + return payload + + +def _query_payload(args: argparse.Namespace) -> dict[str, Any]: + try: + parameters = json.loads(args.parameters) + except json.JSONDecodeError as exc: + raise ValueError(f"graph-query --parameters must be a JSON object: {exc}") from exc + if not isinstance(parameters, dict): + raise ValueError("graph-query --parameters must be a JSON object") + return {"statement": args.statement, "parameters": parameters, "limit": args.limit} + + +def add_json_output_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--pretty", action="store_true", help="Emit indented JSON output") + + +def add_compact_context_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--limit", type=int, default=3, help="Maximum search hits to return") + parser.add_argument("--profile", choices=sorted(CONTEXT_PROFILES), default="brief", help="Context profile") + parser.add_argument("--budget", type=int, default=600, help="Approximate per-hit context character budget") + parser.add_argument("--max-depth", type=int, default=None, help="Override the context profile depth") + parser.add_argument("--context-limit", type=int, default=3, help="Maximum context items per search hit") + parser.add_argument("--detail", choices=sorted(DETAIL_LEVELS), default="standard", help="Output detail level") + parser.add_argument("--format", choices=("json", "block"), default="json", help="Output format") + add_json_output_arguments(parser) + + +def add_runtime_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--repo-root", default=".", help="Repository root containing .codebaseGraph/config.json") + parser.add_argument("--config", default=None, help="Path to .codebaseGraph/config.json") + parser.add_argument("--db", default=None, help="Override LadyBugDB path") + parser.add_argument("--manifest", default=None, help="Override manifest path") + + +def add_graph_compatibility_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--no-refresh", action="store_true", help="Accepted for search/context command parity") + parser.add_argument("--json", action="store_true", help="Accepted for search/context command parity; same as --format json") + + +def _add_graph_health_arguments(parser: argparse.ArgumentParser) -> None: + add_runtime_arguments(parser) + add_json_output_arguments(parser) + + +def _add_graph_search_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument("query", help="Search query") + add_compact_context_arguments(parser) + add_runtime_arguments(parser) + add_graph_compatibility_arguments(parser) + + +def _add_graph_context_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument("query", nargs="?", help="Search query") + parser.add_argument("--node-id", default=None, help="Explicit graph node id") + parser.add_argument("--node-type", default=None, help="Explicit graph node type") + add_compact_context_arguments(parser) + add_runtime_arguments(parser) + add_graph_compatibility_arguments(parser) + + +def _add_graph_architecture_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--group", default=None, help="Optional architecture query group") + add_json_output_arguments(parser) + + +def _add_graph_query_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument("statement", help="Read-only graph query statement") + parser.add_argument("--parameters", default="{}", help="JSON object with query parameters") + parser.add_argument("--limit", type=int, default=100, help="Maximum rows to return") + add_runtime_arguments(parser) + add_json_output_arguments(parser) + + +def _object_schema( + properties: dict[str, Any] | None = None, + *, + required: Sequence[str] = (), +) -> dict[str, Any]: + schema: dict[str, Any] = { + "type": "object", + "properties": properties or {}, + "additionalProperties": False, + } + if required: + schema["required"] = list(required) + return schema + + +def _search_schema(*, required: Sequence[str]) -> dict[str, Any]: + return _object_schema( + { + "query": {"type": "string"}, + "limit": {"type": "integer", "minimum": 1}, + "profile": {"type": "string"}, + "budget": {"type": "integer", "minimum": 0}, + "max_depth": {"type": "integer", "minimum": 0}, + "context_limit": {"type": "integer", "minimum": 0}, + "detail": {"type": "string", "enum": sorted(DETAIL_LEVELS)}, + "output_format": {"type": "string", "enum": ["json", "block"]}, + "node_id": {"type": "string"}, + "node_type": {"type": "string"}, + }, + required=required, + ) + + +GRAPH_COMMAND_SPECS = ( + GraphCommandSpec( + command_name="graph-health", + tool_name="graph_health", + help="Check configured graph paths", + description="Check the configured codebaseGraph database path and manifest path.", + input_schema=_object_schema(), + add_arguments=_add_graph_health_arguments, + payload_from_args=_empty_payload, + ), + GraphCommandSpec( + command_name="graph-search", + tool_name="graph_search", + help="Search the code graph with compact context", + description="Search code, documentation, paths, and dependencies with compact graph context.", + input_schema=_search_schema(required=("query",)), + add_arguments=_add_graph_search_arguments, + payload_from_args=search_arguments_payload, + ), + GraphCommandSpec( + command_name="graph-context", + tool_name="graph_context", + help="Return compact graph context", + description="Return compact context for a search query or explicit node_id/node_type pair.", + input_schema=_search_schema(required=()), + add_arguments=_add_graph_context_arguments, + payload_from_args=_context_payload, + ), + GraphCommandSpec( + command_name="graph-schema", + tool_name="graph_schema", + help="Return ontology schema, indexes, profiles, and helpers", + description="Return ontology schema, search indexes, context profiles, and query helper metadata.", + input_schema=_object_schema(), + add_arguments=add_json_output_arguments, + payload_from_args=_empty_payload, + requires_runtime=False, + ), + GraphCommandSpec( + command_name="graph-query-helpers", + tool_name="graph_query_helpers", + help="Return named read-only graph query helpers", + description="Return named read-only query helpers for common graph exploration tasks.", + input_schema=_object_schema(), + add_arguments=add_json_output_arguments, + payload_from_args=_empty_payload, + requires_runtime=False, + ), + GraphCommandSpec( + command_name="graph-architecture-queries", + tool_name="graph_architecture_queries", + help="Return the architecture-discovery query catalog", + description="Return the grouped architecture-discovery Cypher catalog for coding-agent first-step orientation.", + input_schema=_object_schema( + { + "group": { + "type": "string", + "description": "Optional architecture query group to return.", + }, + } + ), + add_arguments=_add_graph_architecture_arguments, + payload_from_args=_architecture_payload, + requires_runtime=False, + ), + GraphCommandSpec( + command_name="graph-query", + tool_name="graph_query", + help="Execute a restricted read-only graph query", + description="Execute a restricted read-only graph query against the configured database.", + input_schema=_object_schema( + { + "statement": {"type": "string"}, + "parameters": {"type": "object"}, + "limit": {"type": "integer", "minimum": 1, "maximum": MAX_GRAPH_QUERY_LIMIT}, + }, + required=("statement",), + ), + add_arguments=_add_graph_query_arguments, + payload_from_args=_query_payload, + ), +) + diff --git a/src/codebase_graph/mcp/tools.py b/src/codebase_graph/mcp/tools.py index 796b533..0d98722 100644 --- a/src/codebase_graph/mcp/tools.py +++ b/src/codebase_graph/mcp/tools.py @@ -8,8 +8,9 @@ from codebase_graph.diagnostics import log_event from codebase_graph.ontology import QUERY_HELPERS, schema_payload from codebase_graph.reasoning import CompactContextBuilder, architecture_query_catalog -from codebase_graph.retrieval import DETAIL_LEVELS, SearchRequest, SearchService +from codebase_graph.retrieval import DETAIL_LEVELS, SearchRequest, SearchService, serialize_graph_block +from .graph_commands import MAX_GRAPH_QUERY_LIMIT, graph_tool_specs from .runtime import GraphRuntimeConfig, open_graph_store READ_ONLY_DENY_RE = re.compile( @@ -19,14 +20,13 @@ r")\b", re.IGNORECASE, ) -MAX_GRAPH_QUERY_LIMIT = 1000 class UnknownToolError(ValueError): pass -def handle_tool_call(name: str, arguments: dict[str, Any], *, runtime: GraphRuntimeConfig) -> dict[str, Any]: +def handle_tool_call(name: str, arguments: dict[str, Any], *, runtime: GraphRuntimeConfig | None) -> dict[str, Any]: if name == "graph_health": return _health(runtime) if name == "graph_schema": @@ -36,14 +36,14 @@ def handle_tool_call(name: str, arguments: dict[str, Any], *, runtime: GraphRunt if name == "graph_architecture_queries": return architecture_query_catalog(group=_optional_str(arguments.get("group"))) if name == "graph_search": - with open_graph_store(runtime) as store: + with open_graph_store(_require_runtime(runtime, name)) as store: request = _search_request(arguments) return SearchService(store).search(request).as_dict(detail=request.detail) if name == "graph_context": - with open_graph_store(runtime) as store: + with open_graph_store(_require_runtime(runtime, name)) as store: return _context_payload(store, arguments) if name == "graph_query": - with open_graph_store(runtime) as store: + with open_graph_store(_require_runtime(runtime, name)) as store: return _query_payload(store, arguments) raise UnknownToolError(f"Unknown codebaseGraph MCP tool: {name}") @@ -51,16 +51,25 @@ def handle_tool_call(name: str, arguments: dict[str, Any], *, runtime: GraphRunt def call_tool_result(name: str, arguments: dict[str, Any], *, runtime: GraphRuntimeConfig) -> dict[str, Any]: try: payload = handle_tool_call(name, arguments, runtime=runtime) + return tool_result(name, payload, arguments) except UnknownToolError: raise except Exception as exc: return tool_error_result(name, exc) - return tool_result(payload) -def tool_result(payload: dict[str, Any]) -> dict[str, Any]: +def _require_runtime(runtime: GraphRuntimeConfig | None, tool_name: str) -> GraphRuntimeConfig: + if runtime is None: + raise ValueError(f"{tool_name} requires a graph runtime") + return runtime + + +def tool_result(name: str, payload: dict[str, Any], arguments: dict[str, Any] | None = None) -> dict[str, Any]: + text = json.dumps(payload, separators=(",", ":"), sort_keys=True) + if name in {"graph_search", "graph_context"} and _output_format(arguments or {}) == "block": + text = serialize_graph_block(payload) return { - "content": [{"type": "text", "text": json.dumps(payload, separators=(",", ":"), sort_keys=True)}], + "content": [{"type": "text", "text": text}], "structuredContent": payload, "isError": False, } @@ -89,61 +98,7 @@ def tool_error_result(name: str, exc: Exception) -> dict[str, Any]: def tool_specs() -> list[dict[str, Any]]: - return [ - { - "name": "graph_health", - "description": "Check the configured codebaseGraph database path and manifest path.", - "inputSchema": {"type": "object", "properties": {}, "additionalProperties": False}, - }, - { - "name": "graph_search", - "description": "Search code, documentation, paths, and dependencies with compact graph context.", - "inputSchema": _search_schema(required=("query",)), - }, - { - "name": "graph_context", - "description": "Return compact context for a search query or explicit node_id/node_type pair.", - "inputSchema": _search_schema(required=()), - }, - { - "name": "graph_schema", - "description": "Return ontology schema, search indexes, context profiles, and query helper metadata.", - "inputSchema": {"type": "object", "properties": {}, "additionalProperties": False}, - }, - { - "name": "graph_query_helpers", - "description": "Return named read-only query helpers for common graph exploration tasks.", - "inputSchema": {"type": "object", "properties": {}, "additionalProperties": False}, - }, - { - "name": "graph_architecture_queries", - "description": "Return the grouped architecture-discovery Cypher catalog for coding-agent first-step orientation.", - "inputSchema": { - "type": "object", - "properties": { - "group": { - "type": "string", - "description": "Optional architecture query group to return.", - }, - }, - "additionalProperties": False, - }, - }, - { - "name": "graph_query", - "description": "Execute a restricted read-only graph query against the configured database.", - "inputSchema": { - "type": "object", - "properties": { - "statement": {"type": "string"}, - "parameters": {"type": "object"}, - "limit": {"type": "integer", "minimum": 1, "maximum": MAX_GRAPH_QUERY_LIMIT}, - }, - "required": ["statement"], - "additionalProperties": False, - }, - }, - ] + return graph_tool_specs() def _health(runtime: GraphRuntimeConfig) -> dict[str, Any]: @@ -275,25 +230,6 @@ def _json_safe(value: Any) -> Any: return str(value) -def _search_schema(*, required: tuple[str, ...]) -> dict[str, Any]: - return { - "type": "object", - "properties": { - "query": {"type": "string"}, - "limit": {"type": "integer", "minimum": 1}, - "profile": {"type": "string"}, - "budget": {"type": "integer", "minimum": 0}, - "max_depth": {"type": "integer", "minimum": 0}, - "context_limit": {"type": "integer", "minimum": 0}, - "detail": {"type": "string", "enum": sorted(DETAIL_LEVELS)}, - "node_id": {"type": "string"}, - "node_type": {"type": "string"}, - }, - "required": list(required), - "additionalProperties": False, - } - - def _optional_int(value: Any) -> int | None: if value is None or value == "": return None @@ -312,3 +248,10 @@ def _detail(arguments: dict[str, Any]) -> str: valid = ", ".join(sorted(DETAIL_LEVELS)) raise ValueError(f"Unknown detail level: {detail}. Valid levels: {valid}") return detail + + +def _output_format(arguments: dict[str, Any]) -> str: + output_format = str(arguments.get("output_format", "json")) + if output_format not in {"json", "block"}: + raise ValueError(f"Unknown output format: {output_format}. Valid formats: block, json") + return output_format diff --git a/src/codebase_graph/retrieval/__init__.py b/src/codebase_graph/retrieval/__init__.py index 09fe35e..761622c 100644 --- a/src/codebase_graph/retrieval/__init__.py +++ b/src/codebase_graph/retrieval/__init__.py @@ -5,6 +5,9 @@ intentional_summary_omissions, parse_search_block, serialize_agent_search_block, + serialize_context_block, + serialize_graph_block, + serialize_parseable_search_block, serialize_search_block, ) from .search import DETAIL_LEVELS, CompactContextPayload, SearchHit, SearchRequest, SearchService @@ -19,5 +22,8 @@ "intentional_summary_omissions", "parse_search_block", "serialize_agent_search_block", + "serialize_context_block", + "serialize_graph_block", + "serialize_parseable_search_block", "serialize_search_block", ] diff --git a/src/codebase_graph/retrieval/block_format.py b/src/codebase_graph/retrieval/block_format.py index d4c15d6..10c4020 100644 --- a/src/codebase_graph/retrieval/block_format.py +++ b/src/codebase_graph/retrieval/block_format.py @@ -11,8 +11,8 @@ ONTOLOGY_TERMS = {"Class", "Method", "Scope", "Contains", "outgoing", "path", "span", "id", "label", "rank_score"} -def serialize_search_block(payload: Mapping[str, Any]) -> str: - """Serialize graph-search JSON into a readable ontology-preserving block format.""" +def serialize_parseable_search_block(payload: Mapping[str, Any]) -> str: + """Serialize graph-search JSON into a parseable debug block format.""" lines = [ " | ".join( [ @@ -73,7 +73,7 @@ def serialize_search_block(payload: Mapping[str, Any]) -> str: def serialize_agent_search_block(payload: Mapping[str, Any]) -> str: - """Serialize graph-search JSON into a more aggressive display-only agent block.""" + """Serialize graph-search JSON into the compact runtime block format.""" lines = [f"q {_format_value(str(payload.get('query', '')))}"] current_path: str | None = None result_keys = {_record_key(result) for result in payload.get("results", [])} @@ -121,6 +121,49 @@ def serialize_agent_search_block(payload: Mapping[str, Any]) -> str: return "\n".join(lines) + "\n" +def serialize_context_block(payload: Mapping[str, Any]) -> str: + """Serialize an explicit graph-context payload into a readable block.""" + header = [ + f"context {payload.get('node_type', '')}", + f"id={_format_value(str(payload.get('node_id', '')))}", + f"profile={_format_value(str(payload.get('profile', '')))}", + ] + lines = [" ".join(header)] + current_path: str | None = None + for context in payload.get("context", []): + context_path = str(context.get("path", "")) + if context_path != current_path: + if len(lines) > 1: + lines.append("") + lines.append(f"file path {_format_value(context_path)}") + current_path = context_path + context_parts = [ + f" {context.get('direction', '')}", + str(context.get("relation", "")), + str(context.get("type", "")), + _format_value(str(context.get("label", ""))), + _format_span(_span(context.get("span", {}))), + ] + context_summary = _meaningful_summary(context) + if context_summary: + context_parts.append(f"summary={_format_value(context_summary)}") + lines.append(" ".join(context_parts)) + return "\n".join(lines) + "\n" + + +def serialize_graph_block(payload: Mapping[str, Any]) -> str: + if "results" in payload: + return serialize_agent_search_block(payload) + if "context" in payload and "node_id" in payload and "node_type" in payload: + return serialize_context_block(payload) + raise ValueError("Block format is only supported for graph-search and graph-context payloads") + + +def serialize_search_block(payload: Mapping[str, Any]) -> str: + """Backward-compatible alias for the parseable debug block format.""" + return serialize_parseable_search_block(payload) + + def canonicalize_search_payload(payload: Mapping[str, Any]) -> dict[str, Any]: records: list[dict[str, Any]] = [] for result in payload.get("results", []): @@ -316,6 +359,9 @@ def _record_key(record: Mapping[str, Any]) -> tuple[str, str, str, tuple[tuple[s "canonicalize_search_payload", "intentional_summary_omissions", "parse_search_block", + "serialize_context_block", "serialize_agent_search_block", + "serialize_graph_block", + "serialize_parseable_search_block", "serialize_search_block", ] diff --git a/src/codebase_graph/setup/installer.py b/src/codebase_graph/setup/installer.py index 2799b94..340ec10 100644 --- a/src/codebase_graph/setup/installer.py +++ b/src/codebase_graph/setup/installer.py @@ -7,7 +7,7 @@ import subprocess from dataclasses import dataclass from pathlib import Path -from typing import Any +from typing import Any, Callable from codebase_graph.mcp.protocol import LATEST_PROTOCOL_VERSION @@ -15,14 +15,9 @@ from .descriptor import McpServerDescriptor, build_server_descriptor from .state import MCP_SERVER_NAME, load_setup_config -INSTALL_CLIENTS = ("codex", "claude", "claude-project", "lmstudio", "hermes", "openclaw", "generic") SCOPES = ("local", "user", "project") -NATIVE_EXECUTABLES = { - "codex": "codex", - "claude": "claude", - "claude-project": "claude", - "openclaw": "openclaw", -} +NativeCommandBuilder = Callable[[McpServerDescriptor, str], list[str]] +VisibilityCommandBuilder = Callable[[], list[str]] @dataclass(frozen=True, slots=True) @@ -84,6 +79,94 @@ def as_dict(self) -> dict[str, Any]: return payload +@dataclass(frozen=True, slots=True) +class InstallClientStrategy: + client_id: str + adapter_id: str | None = None + project_adapter_id: str | None = None + forced_scope: str | None = None + native_executable: str | None = None + native_command_builder: NativeCommandBuilder | None = None + visibility_command_builder: VisibilityCommandBuilder | None = None + + def install_scope(self, scope: str) -> str: + return self.forced_scope or scope + + def adapter_client_id(self, scope: str) -> str: + if self.project_adapter_id is not None and self.install_scope(scope) == "project": + return self.project_adapter_id + return self.adapter_id or self.client_id + + def native_command(self, descriptor: McpServerDescriptor, *, scope: str) -> list[str] | None: + if self.native_command_builder is None: + return None + return self.native_command_builder(descriptor, self.install_scope(scope)) + + def visibility_command(self) -> list[str] | None: + if self.visibility_command_builder is None: + return None + return self.visibility_command_builder() + + +def _codex_native_command(descriptor: McpServerDescriptor, scope: str) -> list[str]: + return ["codex", "mcp", "add", descriptor.name, "--", descriptor.command, *descriptor.args] + + +def _claude_native_command(descriptor: McpServerDescriptor, scope: str) -> list[str]: + return [ + "claude", + "mcp", + "add", + "--transport", + "stdio", + "--scope", + scope, + descriptor.name, + "--", + descriptor.command, + *descriptor.args, + ] + + +def _openclaw_native_command(descriptor: McpServerDescriptor, scope: str) -> list[str]: + entry = descriptor.stdio_entry(include_type=True) + return ["openclaw", "mcp", "set", descriptor.name, json.dumps(entry, separators=(",", ":"), sort_keys=True)] + + +INSTALL_STRATEGIES: dict[str, InstallClientStrategy] = { + "codex": InstallClientStrategy( + client_id="codex", + native_executable="codex", + native_command_builder=_codex_native_command, + visibility_command_builder=lambda: ["codex", "mcp", "list"], + ), + "claude": InstallClientStrategy( + client_id="claude", + project_adapter_id="claude-project", + native_executable="claude", + native_command_builder=_claude_native_command, + visibility_command_builder=lambda: ["claude", "mcp", "list"], + ), + "claude-project": InstallClientStrategy( + client_id="claude-project", + forced_scope="project", + native_executable="claude", + native_command_builder=_claude_native_command, + visibility_command_builder=lambda: ["claude", "mcp", "list"], + ), + "lmstudio": InstallClientStrategy(client_id="lmstudio"), + "hermes": InstallClientStrategy(client_id="hermes"), + "openclaw": InstallClientStrategy( + client_id="openclaw", + native_executable="openclaw", + native_command_builder=_openclaw_native_command, + visibility_command_builder=lambda: ["openclaw", "mcp", "list"], + ), + "generic": InstallClientStrategy(client_id="generic"), +} +INSTALL_CLIENTS = tuple(INSTALL_STRATEGIES) + + def supported_install_client_ids(*, include_all: bool = False) -> tuple[str, ...]: values = [*INSTALL_CLIENTS] if include_all: @@ -104,6 +187,7 @@ def install_mcp_clients(options: McpInstallOptions) -> list[McpInstallResult]: def install_mcp_server(options: McpInstallOptions) -> McpInstallResult: _validate_options(options) + strategy = _client_strategy(options.client) descriptor = _build_descriptor(options) entry = descriptor.stdio_entry() if options.skip or options.client == "none": @@ -119,12 +203,13 @@ def install_mcp_server(options: McpInstallOptions) -> McpInstallResult: entry=entry, ) - native_command = _native_command(options.client, descriptor, scope=options.scope) + native_command = strategy.native_command(descriptor, scope=options.scope) use_native = ( options.prefer_native and options.client_config_path is None and native_command is not None - and shutil.which(_native_executable(options.client)) + and strategy.native_executable is not None + and shutil.which(strategy.native_executable) ) if options.dry_run: if use_native: @@ -156,14 +241,14 @@ def install_mcp_server(options: McpInstallOptions) -> McpInstallResult: descriptor, dry_run=False, native_command=native_command, - native_error=_missing_native_error(options.client) if native_command is not None else None, + native_error=_missing_native_error(strategy) if native_command is not None else None, ) def _install_with_failure_result(options: McpInstallOptions, client: str) -> McpInstallResult: client_options = McpInstallOptions( client=client, - scope=_scope_for_client(client, options.scope), + scope=_client_strategy(client).install_scope(options.scope), setup_config_path=options.setup_config_path, server_name=options.server_name, client_config_path=options.client_config_path, @@ -207,7 +292,7 @@ def _file_adapter_result( native_command: list[str] | None = None, native_error: str | None = None, ) -> McpInstallResult: - adapter = get_client_adapter(_adapter_client_id(options.client, options.scope)) + adapter = get_client_adapter(_client_strategy(options.client).adapter_client_id(options.scope)) path = ( Path(options.client_config_path).expanduser().resolve() if options.client_config_path is not None @@ -348,7 +433,7 @@ def _verify_stdio(descriptor: McpServerDescriptor, *, timeout: int) -> dict[str, def _verify_client_visibility(client: str, server_name: str, *, timeout: int) -> dict[str, Any]: - command = _visibility_command(client) + command = _client_strategy(client).visibility_command() if command is None: return {"ok": True, "skipped": True, "reason": f"{client} has no CLI visibility check"} executable = command[0] @@ -418,39 +503,6 @@ def _frame_json_rpc(method: str, params: dict[str, Any], *, request_id: int) -> return f"Content-Length: {len(body)}\r\n\r\n".encode("ascii") + body -def _native_command(client: str, descriptor: McpServerDescriptor, *, scope: str) -> list[str] | None: - if client == "codex": - return ["codex", "mcp", "add", descriptor.name, "--", descriptor.command, *descriptor.args] - if client in {"claude", "claude-project"}: - return [ - "claude", - "mcp", - "add", - "--transport", - "stdio", - "--scope", - _scope_for_client(client, scope), - descriptor.name, - "--", - descriptor.command, - *descriptor.args, - ] - if client == "openclaw": - entry = descriptor.stdio_entry(include_type=True) - return ["openclaw", "mcp", "set", descriptor.name, json.dumps(entry, separators=(",", ":"), sort_keys=True)] - return None - - -def _visibility_command(client: str) -> list[str] | None: - if client == "codex": - return ["codex", "mcp", "list"] - if client in {"claude", "claude-project"}: - return ["claude", "mcp", "list"] - if client == "openclaw": - return ["openclaw", "mcp", "list"] - return None - - def _build_descriptor(options: McpInstallOptions) -> McpServerDescriptor: config_path = Path(options.setup_config_path).expanduser().resolve() repo_root: Path | None = None @@ -476,27 +528,16 @@ def _validate_options(options: McpInstallOptions) -> None: raise ValueError(f"Unsupported MCP install scope: {options.scope}. Supported scopes: {', '.join(SCOPES)}") -def _native_executable(client: str) -> str: - return NATIVE_EXECUTABLES[client] - - -def _adapter_client_id(client: str, scope: str) -> str: - if client == "claude" and scope == "project": - return "claude-project" - return client - - -def _scope_for_client(client: str, scope: str) -> str: - if client == "claude-project": - return "project" - return scope +def _client_strategy(client: str) -> InstallClientStrategy: + if client == "none": + return InstallClientStrategy(client_id="none") + return INSTALL_STRATEGIES[client] -def _missing_native_error(client: str) -> str | None: - executable = NATIVE_EXECUTABLES.get(client) - if executable is None: +def _missing_native_error(strategy: InstallClientStrategy) -> str | None: + if strategy.native_executable is None: return None - return f"{executable} executable not found" + return f"{strategy.native_executable} executable not found" def _subprocess_error(completed: subprocess.CompletedProcess[str]) -> str: diff --git a/src/codebase_graph/setup/instructions.py b/src/codebase_graph/setup/instructions.py index 183ad33..98664c7 100644 --- a/src/codebase_graph/setup/instructions.py +++ b/src/codebase_graph/setup/instructions.py @@ -76,9 +76,10 @@ def _instruction_block(*, server_name: str, config_path: Path, setup_command: st return ( f"{START_MARKER}\n" "## codebaseGraph workflow\n" - "- Treat the repo-local `.codebaseGraph` graph as the project operating source of truth.\n" - f"- Use `{setup_command} graph-search --repo-root . --no-refresh --detail slim --context-limit 1 --json` before answering repo-structure questions or performing coding tasks.\n" - f"- Use `{setup_command} graph-context --repo-root . --profile --no-refresh --detail slim --context-limit 2 --json` when relationships or nearby evidence matter; useful profiles include `definitions`, `dependencies`, `callgraph`, `docs`, `runtime`, and `change_impact`.\n" + "- Treat the repo-local `.codebaseGraph` graph as the project operating source of truth. It is prohibited to read the code source before you find the target files using the graph.\n" + "- AI agents must use block format for `graph-search` and `graph-context`; reserve `--json` for tests, APIs, or explicit structured-payload debugging.\n" + f"- Use `{setup_command} graph-search --repo-root . --no-refresh --detail slim --context-limit 1 --format block` before answering repo-structure questions or performing coding tasks.\n" + f"- Use `{setup_command} graph-context --repo-root . --profile --no-refresh --detail slim --context-limit 2 --format block` when relationships or nearby evidence matter; useful profiles include `definitions`, `dependencies`, `callgraph`, `docs`, `runtime`, and `change_impact`.\n" f"- For architecture orientation, run `{setup_command} graph-architecture-queries`, then execute selected read-only statements with `{setup_command} graph-query \"\" --repo-root .`.\n" f"- Use `{setup_command} graph-schema` or `{setup_command} graph-query-helpers` before writing raw graph queries, add `--pretty` for indented JSON when humans need to inspect output, and keep `graph-query` read-only.\n" f"- Refresh the graph with `{setup_command} setup --repo-root . --mcp-client none` when files change materially. Setup config: `{config_path.as_posix()}`.\n" diff --git a/src/codebase_graph/setup/orchestrator.py b/src/codebase_graph/setup/orchestrator.py index 59bf6a5..d7c667f 100644 --- a/src/codebase_graph/setup/orchestrator.py +++ b/src/codebase_graph/setup/orchestrator.py @@ -151,19 +151,10 @@ def run_setup(options: SetupOptions) -> SetupResult: def _materialization_payload(result: Any) -> dict[str, Any]: - return { - "mode": getattr(result, "mode"), - "scanned": getattr(result, "scanned"), - "rebuilt": getattr(result, "rebuilt"), - "skipped": getattr(result, "skipped"), - "deleted": getattr(result, "deleted"), - "diagnostics": list(getattr(result, "diagnostics")), - "manifest_path": getattr(result, "manifest_path"), - "rebuilt_paths": list(getattr(result, "rebuilt_paths")), - "skipped_paths": list(getattr(result, "skipped_paths")), - "deleted_paths": list(getattr(result, "deleted_paths")), - "graph_summary": dict(getattr(result, "graph_summary")), - } + as_dict = getattr(result, "as_dict", None) + if callable(as_dict): + return as_dict() + raise TypeError(f"Unsupported materialization result: {type(result).__name__}") def _dry_run_materialization(paths: SetupPaths) -> Any: @@ -202,6 +193,21 @@ class _DryRunMaterialization: deleted_paths: tuple[str, ...] = () graph_summary: dict[str, Any] = field(default_factory=dict) + def as_dict(self) -> dict[str, Any]: + return { + "mode": self.mode, + "scanned": self.scanned, + "rebuilt": self.rebuilt, + "skipped": self.skipped, + "deleted": self.deleted, + "diagnostics": list(self.diagnostics), + "manifest_path": self.manifest_path, + "rebuilt_paths": list(self.rebuilt_paths), + "skipped_paths": list(self.skipped_paths), + "deleted_paths": list(self.deleted_paths), + "graph_summary": dict(self.graph_summary), + } + def _config_would_change(path: Path, payload: dict[str, Any]) -> bool: if not path.exists(): diff --git a/tests/test_graph_output_block_format.py b/tests/test_graph_output_block_format.py index 77ac6fd..6a8f38a 100644 --- a/tests/test_graph_output_block_format.py +++ b/tests/test_graph_output_block_format.py @@ -11,6 +11,8 @@ canonicalize_search_payload, parse_search_block, serialize_agent_search_block, + serialize_context_block, + serialize_parseable_search_block, serialize_search_block, ) @@ -32,9 +34,10 @@ def test_token_counting_uses_encoded_text_length() -> None: def test_raw_vs_block_comparison_preserves_search_service_fixture() -> None: payload = json.loads(FIXTURE_PATH.read_text(encoding="utf-8")) - block = serialize_search_block(payload) + block = serialize_parseable_search_block(payload) assert parse_search_block(block) == canonicalize_search_payload(payload) + assert serialize_search_block(payload) == block def test_l_same_is_only_emitted_for_matching_context_spans() -> None: @@ -122,6 +125,30 @@ def test_agent_block_reduces_display_only_boilerplate() -> None: ) in block +def test_context_block_serializes_explicit_node_context() -> None: + block = serialize_context_block( + { + "node_id": "Class:943d6556d328f1c7ca67", + "node_type": "Class", + "profile": "definitions", + "context": [ + { + "direction": "outgoing", + "relation": "Contains", + "type": "Method", + "label": "search", + "path": "src/codebase_graph/retrieval/search.py", + "span": {"line_start": 123, "line_end": 149}, + } + ], + } + ) + + assert block.startswith("context Class id=Class:943d6556d328f1c7ca67 profile=definitions") + assert "file path src/codebase_graph/retrieval/search.py" in block + assert "outgoing Contains Method search L123-L149" in block + + def _load_benchmark_script() -> Any: spec = importlib.util.spec_from_file_location("compare_graph_output_tokens", SCRIPT_PATH) assert spec is not None diff --git a/tests/test_mcp_installer.py b/tests/test_mcp_installer.py index b238607..7809422 100644 --- a/tests/test_mcp_installer.py +++ b/tests/test_mcp_installer.py @@ -10,6 +10,8 @@ from codebase_graph.setup.clients import get_client_adapter from codebase_graph.setup.descriptor import build_server_descriptor from codebase_graph.setup.installer import ( + INSTALL_CLIENTS, + INSTALL_STRATEGIES, McpInstallOptions, default_server_name, install_mcp_clients, @@ -22,6 +24,18 @@ def test_default_server_name_is_namespace_safe() -> None: assert default_server_name("My Service") == "codebase_graph_my_service" +def test_install_strategy_registry_covers_advertised_clients() -> None: + assert set(INSTALL_CLIENTS) == set(INSTALL_STRATEGIES) + for client, strategy in INSTALL_STRATEGIES.items(): + assert strategy.adapter_client_id("local") + if strategy.native_command_builder is not None: + assert strategy.native_executable + if client == "claude": + assert strategy.adapter_client_id("project") == "claude-project" + if client == "claude-project": + assert strategy.install_scope("local") == "project" + + def test_codex_native_command_generation_uses_repo_server_name( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/test_mcp_portability.py b/tests/test_mcp_portability.py index ec44dcc..f13eecd 100644 --- a/tests/test_mcp_portability.py +++ b/tests/test_mcp_portability.py @@ -186,6 +186,12 @@ def test_stdio_mcp_wire_initialize_list_call_and_tool_error(tmp_path: Path) -> N "tools/call", {"name": "graph_search", "arguments": {"query": "SampleService", "limit": 2}}, ) + block_search = _rpc( + proc.stdin, + proc.stdout, + "tools/call", + {"name": "graph_search", "arguments": {"query": "SampleService", "limit": 2, "output_format": "block"}}, + ) failure = _rpc( proc.stdin, proc.stdout, @@ -201,9 +207,13 @@ def test_stdio_mcp_wire_initialize_list_call_and_tool_error(tmp_path: Path) -> N graph_search_tool = next(tool for tool in listed["result"]["tools"] if tool["name"] == "graph_search") assert "context_limit" in graph_search_tool["inputSchema"]["properties"] assert graph_search_tool["inputSchema"]["properties"]["detail"]["enum"] == ["slim", "standard"] + assert graph_search_tool["inputSchema"]["properties"]["output_format"]["enum"] == ["json", "block"] assert health["result"]["structuredContent"]["ok"] is True assert search["result"]["structuredContent"]["results"] assert "\n " not in search["result"]["content"][0]["text"] + assert block_search["result"]["structuredContent"] == search["result"]["structuredContent"] + assert block_search["result"]["content"][0]["text"].startswith("q SampleService\n") + assert "id=Class:" in block_search["result"]["content"][0]["text"] assert "error" not in failure assert failure["result"]["isError"] is True assert failure["result"]["structuredContent"]["error"]["type"] == "ValueError" diff --git a/tests/test_search.py b/tests/test_search.py index efa5787..1e76066 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -7,11 +7,12 @@ import pytest -from codebase_graph.cli import main as cli_main +from codebase_graph.cli import _build_parser, main as cli_main from codebase_graph.db import GraphNeighbor, SearchIndexRow from codebase_graph.ingest import GraphMaterializer +from codebase_graph.mcp.graph_commands import graph_command_spec, graph_tool_specs from codebase_graph.mcp.runtime import GraphRuntimeConfig -from codebase_graph.mcp.tools import MAX_GRAPH_QUERY_LIMIT, _query_payload, handle_tool_call +from codebase_graph.mcp.tools import MAX_GRAPH_QUERY_LIMIT, _query_payload, handle_tool_call, tool_specs from codebase_graph.reasoning import CompactContextBuilder, ContextNode from codebase_graph.retrieval.search import CompactContextPayload, SearchHit, SearchRequest, SearchService @@ -508,6 +509,31 @@ def test_cli_graph_commands_match_mcp_tool_payloads(tmp_path: Path, capsys: pyte assert "score" not in search_payload["results"][0] assert len(search_payload["results"][0].get("context", [])) <= 1 + assert cli_main([ + "graph-search", + "SampleService", + "--repo-root", + source_root.as_posix(), + "--db", + db_path.as_posix(), + "--manifest", + manifest_path.as_posix(), + "--limit", + "2", + "--context-limit", + "1", + "--detail", + "slim", + "--no-refresh", + "--format", + "block", + ]) == 0 + block_output = capsys.readouterr().out + assert block_output.startswith("q SampleService\n") + assert "file path sample_project/service.py" in block_output + assert "id=Class:" in block_output + assert not block_output.lstrip().startswith("{") + hit = next(item for item in search_payload["results"] if item["label"] == "SampleService") context_args = { "node_id": hit["id"], @@ -539,6 +565,31 @@ def test_cli_graph_commands_match_mcp_tool_payloads(tmp_path: Path, capsys: pyte ]) == 0 assert json.loads(capsys.readouterr().out) == handle_tool_call("graph_context", context_args, runtime=runtime) + assert cli_main([ + "graph-context", + "--node-id", + hit["id"], + "--node-type", + hit["type"], + "--repo-root", + source_root.as_posix(), + "--db", + db_path.as_posix(), + "--manifest", + manifest_path.as_posix(), + "--profile", + "definitions", + "--limit", + "1", + "--detail", + "slim", + "--format", + "block", + ]) == 0 + context_block = capsys.readouterr().out + assert context_block.startswith(f"context {hit['type']} id={hit['id']} profile=definitions\n") + assert "file path " in context_block + statement = "MATCH (n) RETURN count(n) AS total_nodes LIMIT 1" query_args = {"statement": statement, "parameters": {}, "limit": 5} assert cli_main([ @@ -556,6 +607,80 @@ def test_cli_graph_commands_match_mcp_tool_payloads(tmp_path: Path, capsys: pyte assert json.loads(capsys.readouterr().out) == handle_tool_call("graph_query", query_args, runtime=runtime) +def test_graph_command_specs_drive_mcp_tool_specs() -> None: + assert tool_specs() == graph_tool_specs() + + +def test_graph_command_specs_build_cli_payloads() -> None: + parser = _build_parser() + cases = [ + ( + [ + "graph-search", + "SampleService", + "--limit", + "2", + "--context-limit", + "1", + "--detail", + "slim", + ], + "graph_search", + { + "query": "SampleService", + "limit": 2, + "profile": "brief", + "budget": 600, + "context_limit": 1, + "detail": "slim", + }, + ), + ( + [ + "graph-context", + "--node-id", + "Class:1", + "--node-type", + "Class", + "--profile", + "definitions", + "--limit", + "1", + "--detail", + "slim", + ], + "graph_context", + { + "node_id": "Class:1", + "node_type": "Class", + "limit": 1, + "profile": "definitions", + "budget": 600, + "context_limit": 3, + "detail": "slim", + }, + ), + ( + [ + "graph-query", + "MATCH (n) RETURN n", + "--parameters", + '{"limit": 1}', + "--limit", + "5", + ], + "graph_query", + {"statement": "MATCH (n) RETURN n", "parameters": {"limit": 1}, "limit": 5}, + ), + ] + for argv, tool_name, expected_payload in cases: + args = parser.parse_args(argv) + spec = graph_command_spec(args.command) + + assert spec.tool_name == tool_name + assert spec.payload_from_args(args) == expected_payload + + def test_cli_graph_metadata_commands_do_not_open_graph_db(capsys: pytest.CaptureFixture[str]) -> None: assert cli_main(["graph-schema"]) == 0 schema_output = capsys.readouterr().out diff --git a/tests/test_setup_workflow.py b/tests/test_setup_workflow.py index 3fcda65..5ec3739 100644 --- a/tests/test_setup_workflow.py +++ b/tests/test_setup_workflow.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import re import sys from pathlib import Path @@ -16,7 +17,7 @@ from codebase_graph.mcp.runtime import runtime_config from codebase_graph.mcp.server import McpGraphServer, handle_tool_call from codebase_graph.setup import SetupError, SetupOptions, run_setup -from codebase_graph.setup.instructions import END_MARKER, START_MARKER +from codebase_graph.setup.instructions import END_MARKER, START_MARKER, upsert_instruction_block from codebase_graph.setup.mcp_config import configure_mcp_client, server_entry from codebase_graph.setup.state import build_setup_config, derive_setup_paths, load_setup_config, write_setup_config @@ -58,10 +59,18 @@ def test_setup_cli_creates_state_db_mcp_config_instructions_and_searchable_docs( assert agents_text.count(END_MARKER) == 1 assert "graph-search" in agents_text assert "graph-context" in agents_text + assert "--format block" in agents_text + assert re.search(r"graph-search .*--json", agents_text) is None + assert re.search(r"graph-context .*--json", agents_text) is None + assert "AI agents must use block format" in agents_text assert "graph-architecture-queries" in agents_text assert "MCP server" not in agents_text assert "graph_architecture_queries" not in agents_text assert "graph_query" not in agents_text + assert ( + "It is prohibited to read the code source before you find the target files using the graph." + in agents_text + ) mcp_payload = tomllib.loads(mcp_config_path.read_text(encoding="utf-8")) assert "otherServer" not in mcp_payload.get("mcp_servers", {}) assert mcp_payload["mcp_servers"]["codebase_graph"]["args"] == [ @@ -108,6 +117,26 @@ def test_setup_cli_creates_state_db_mcp_config_instructions_and_searchable_docs( assert any(hit["label"] == "SampleService" for hit in symbol_payload["results"]) +def test_claude_instruction_target_uses_block_format(tmp_path: Path) -> None: + repo_root = tmp_path / "fresh_repo" + repo_root.mkdir() + + result = upsert_instruction_block( + repo_root, + target="claude", + server_name="codebase_graph", + config_path=repo_root / ".codebaseGraph" / "config.json", + ) + claude_text = (repo_root / "CLAUDE.md").read_text(encoding="utf-8") + + assert result.action == "created" + assert result.path == (repo_root / "CLAUDE.md").as_posix() + assert not (repo_root / "AGENTS.md").exists() + assert "--format block" in claude_text + assert re.search(r"graph-search .*--json", claude_text) is None + assert re.search(r"graph-context .*--json", claude_text) is None + + def test_mcp_config_dry_run_preserves_existing_json_servers(tmp_path: Path) -> None: config_path = tmp_path / "mcp.json" config_path.write_text( From b4e2bf12a96aba3b9ce7a799fa4c65fda4199416 Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Thu, 28 May 2026 13:22:04 +0930 Subject: [PATCH 51/53] docs: introduce README capabilities overview The README opener now stays short and follows with a capabilities paragraph that names the graph workflows agents can use before opening source files. Constraint: Keep the addition focused on README introduction only Rejected: Expand the install section again | the request was for capability framing, not setup detail Confidence: high Scope-risk: narrow Directive: Keep capability claims aligned with graph-search, graph-context, architecture-query, and read-only graph-query surfaces Tested: ./.venv/bin/ruff check .; ./.venv/bin/python -m pytest; codebase-graph setup --repo-root . --mcp-client none --instructions-target skip --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 400d9dd..6786208 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,11 @@ `CLAUDE.md`, Markdown, and MDX files into a LadyBugDB-backed graph, then exposes search, compact context, schema, query helpers, and read-only MCP tools. +With the graph in place, agents can search symbols and docs, pull compact context for definitions, dependencies, call +graphs, runtime surfaces, and change-impact questions, inspect schema and architecture-query helpers, and run bounded +read-only graph queries through either the CLI or MCP. Setup also keeps repo-local graph state and agent instructions +aligned so assistants can use graph evidence before opening source files. + Requires Python 3.10+ and a package build that includes `real_ladybug`. ## Quick start From e48442d9e4e172d878e2167b9086d6691b116e24 Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Thu, 28 May 2026 13:39:08 +0930 Subject: [PATCH 52/53] docs: explain codebaseGraph agent advantages Refreshes the README opening to keep the user-provided advantages wording around faster orientation, reduced guesswork, impact-aware changes, and lower token/tool overhead. Constraint: User requested preserving their latest README wording without restoring the previous intro paragraph. Confidence: high Scope-risk: narrow Tested: .venv/bin/python -m pytest Tested: .venv/bin/ruff check . Tested: git diff --check Tested: codebase-graph setup --repo-root . --mcp-client none Not-tested: Rendered Markdown preview --- README.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 6786208..7ab167c 100644 --- a/README.md +++ b/README.md @@ -4,12 +4,11 @@ `CLAUDE.md`, Markdown, and MDX files into a LadyBugDB-backed graph, then exposes search, compact context, schema, query helpers, and read-only MCP tools. -With the graph in place, agents can search symbols and docs, pull compact context for definitions, dependencies, call -graphs, runtime surfaces, and change-impact questions, inspect schema and architecture-query helpers, and run bounded -read-only graph queries through either the CLI or MCP. Setup also keeps repo-local graph state and agent instructions -aligned so assistants can use graph evidence before opening source files. +Using codebaseGraph helps agents orient faster, reduce guesswork, keep prompts focused, and make changes faster with better +impact awareness. Because the graph stores local source, documentation, spans, and relationships together, it gives +AI agents a compact evidence layer for safer edits, architecture review, dependency tracing, and onboarding while reducing token consumption and tool calling. -Requires Python 3.10+ and a package build that includes `real_ladybug`. +Requires Python 3.10+ ## Quick start From be90e0a5b5116b93e2d661ff946781cdf5bde8c2 Mon Sep 17 00:00:00 2001 From: rabii-chaarani Date: Thu, 28 May 2026 13:45:50 +0930 Subject: [PATCH 53/53] refactor: enhance intro --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7ab167c..c8c79a5 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ `CLAUDE.md`, Markdown, and MDX files into a LadyBugDB-backed graph, then exposes search, compact context, schema, query helpers, and read-only MCP tools. -Using codebaseGraph helps agents orient faster, reduce guesswork, keep prompts focused, and make changes faster with better +Using `codebaseGraph` helps agents orient and reason faster, reduce guesswork, keep prompts focused, and make changes with better impact awareness. Because the graph stores local source, documentation, spans, and relationships together, it gives AI agents a compact evidence layer for safer edits, architecture review, dependency tracing, and onboarding while reducing token consumption and tool calling.