From 5c6db127df973e9ee9f747e3a239596c961f77b0 Mon Sep 17 00:00:00 2001 From: Dmitry Teryaev Date: Sat, 6 Jun 2026 16:46:17 +0300 Subject: [PATCH] add FileDeps dataclass, .deps.json sidecar, and determinism test (PR-T1) Foundation for incremental Kuzu rebuild (Tier 2 plan). Every full rebuild now writes a .deps.json sidecar alongside the Kuzu database containing per-file dependency metadata (declared types, injects, extends, calls, annotation usage, overrides, clients, producers) for closure expansion in subsequent incremental runs. Adds FileDeps dataclass, _build_file_deps, _write_dependency_index, _read_dependency_index, DepsIndex. Wires into write_kuzu after _write_meta. 5 tests: determinism, deps-json written/validated/coverage/stale/missing. Implements PR-T1 of plans/active/PLAN-TIER2-INCREMENTAL-REBUILD.md. Co-Authored-By: Claude Opus 4.7 --- build_ast_graph.py | 177 ++++++++++++++++++++++++++ tests/test_incremental_equivalence.py | 156 +++++++++++++++++++++++ 2 files changed, 333 insertions(+) create mode 100644 tests/test_incremental_equivalence.py diff --git a/build_ast_graph.py b/build_ast_graph.py index 0d11db80..8e798899 100644 --- a/build_ast_graph.py +++ b/build_ast_graph.py @@ -401,6 +401,24 @@ class GraphTables: type_role_by_node_id: dict[str, str] = field(default_factory=dict) +# ---------- per-file dependency tracking (sidecar .deps.json) ---------- + + +@dataclass +class FileDeps: + """Per-file dependency metadata for incremental rebuild closure expansion.""" + + ext_hash: str = "" + declares: list[str] = field(default_factory=list) + injects: list[str] = field(default_factory=list) + extends: list[str] = field(default_factory=list) + calls: list[str] = field(default_factory=list) + uses_anno: list[str] = field(default_factory=list) + overrides: list[str] = field(default_factory=list) + declares_clients: list[str] = field(default_factory=list) + declares_producers: list[str] = field(default_factory=list) + + # ---------- file walk (see `path_filtering.iter_java_source_files`) ---------- @@ -2990,6 +3008,164 @@ def _write_meta(conn: kuzu.Connection, tables: GraphTables, source_root: Path) - ) +# ---------- dependency index (sidecar .deps.json) ---------- + + +def _build_file_deps(tables: GraphTables, source_root: Path) -> dict[str, FileDeps]: + """Build per-file dependency metadata from GraphTables for the sidecar index.""" + # node_id -> file_path lookup + node_file: dict[str, str] = {} + for entry in tables.types.values(): + node_file[entry.node_id] = entry.file_path + for m in tables.members: + node_file[m.node_id] = m.file_path + + # node_id -> identifier (type FQN or "TypeFQN#method()" for members) + node_fqn: dict[str, str] = {} + for entry in tables.types.values(): + node_fqn[entry.node_id] = entry.decl.fqn + for m in tables.members: + node_fqn[m.node_id] = f"{m.parent_fqn}#{m.decl.signature}" + + deps: dict[str, FileDeps] = {} + for fp in tables.files: + deps[fp] = FileDeps() + + # ext_hash — read file from disk and hash + for fp in deps: + full = source_root / fp + try: + raw = full.read_bytes() + deps[fp].ext_hash = f"sha256:{hashlib.sha256(raw).hexdigest()}" + except OSError: + deps[fp].ext_hash = "" + + # declares — type FQNs declared in this file + for entry in tables.types.values(): + fp = entry.file_path + if fp in deps: + deps[fp].declares.append(entry.outer_fqn or entry.decl.fqn) + + # injects — FQNs of injected symbols (from injects_rows) + for row in tables.injects_rows: + fp = node_file.get(row.src_id) + if fp and fp in deps: + deps[fp].injects.append(row.dst_fqn) + + # extends — FQNs of extended types (from extends_rows) + for row in tables.extends_rows: + fp = node_file.get(row.src_id) + if fp and fp in deps: + deps[fp].extends.append(row.dst_fqn) + + # calls — callee identifiers (method/type FQNs from calls_rows) + for row in tables.calls_rows: + fp = node_file.get(row.src_id) + if fp and fp in deps: + callee = node_fqn.get(row.dst_id) + if callee is not None: + deps[fp].calls.append(callee) + + # uses_anno — annotation simple names from types + members (deduplicated) + for entry in tables.types.values(): + fp = entry.file_path + if fp in deps: + for anno in entry.decl.annotations: + if anno.name not in deps[fp].uses_anno: + deps[fp].uses_anno.append(anno.name) + for m in tables.members: + fp = m.file_path + if fp in deps: + for anno in m.decl.annotations: + if anno.name not in deps[fp].uses_anno: + deps[fp].uses_anno.append(anno.name) + + # overrides — overridden method FQNs (from overrides_rows) + for row in tables.overrides_rows: + fp = node_file.get(row.src_id) + if fp and fp in deps: + overridden = node_fqn.get(row.dst_id) + if overridden is not None: + deps[fp].overrides.append(overridden) + + # declares_clients — member FQNs declaring HTTP clients + for row in tables.client_rows: + fp = row.filename + if fp in deps: + deps[fp].declares_clients.append(row.member_fqn) + + # declares_producers — member FQNs declaring async producers + for row in tables.producer_rows: + fp = row.filename + if fp in deps: + deps[fp].declares_producers.append(row.member_fqn) + + return deps + + +_DEPS_VERSION = 1 + + +def _write_dependency_index( + db_path: Path, + tables: GraphTables, + source_root: Path, +) -> None: + """Write sidecar .deps.json alongside the Kuzu database.""" + deps = _build_file_deps(tables, source_root) + payload = { + "version": _DEPS_VERSION, + "ontology_version": ONTOLOGY_VERSION, + "files": {fp: asdict(d) for fp, d in sorted(deps.items())}, + } + deps_path = db_path.parent / ".deps.json" + tmp = deps_path.with_suffix(".json.tmp") + tmp.write_text(json.dumps(payload, indent=2)) + tmp.rename(deps_path) + + +@dataclass +class DepsIndex: + version: int + ontology_version: int + files: dict[str, FileDeps] + + +def _read_dependency_index(deps_path: Path) -> DepsIndex | None: + """Read and validate sidecar .deps.json. Returns None on missing/corrupt/stale.""" + if not deps_path.is_file(): + return None + try: + raw = json.loads(deps_path.read_text()) + except (json.JSONDecodeError, OSError): + return None + try: + if raw.get("version") != _DEPS_VERSION: + return None + if raw.get("ontology_version") != ONTOLOGY_VERSION: + return None + files: dict[str, FileDeps] = {} + for fp, obj in raw.get("files", {}).items(): + files[fp] = FileDeps( + ext_hash=obj.get("ext_hash", ""), + declares=obj.get("declares", []), + injects=obj.get("injects", []), + extends=obj.get("extends", []), + calls=obj.get("calls", []), + uses_anno=obj.get("uses_anno", []), + overrides=obj.get("overrides", []), + declares_clients=obj.get("declares_clients", []), + declares_producers=obj.get("declares_producers", []), + ) + except Exception: + return None + return DepsIndex( + version=raw["version"], + ontology_version=raw["ontology_version"], + files=files, + ) + + def write_kuzu( db_path: Path, tables: GraphTables, @@ -3031,6 +3207,7 @@ def write_kuzu( _verbose_stderr_line(f"[graph] writing · routes/exposes written in {time.time() - t2:.2f}s") _write_meta(conn, tables, source_root) conn.close() + _write_dependency_index(db_path, tables, source_root) # ---------- CLI ---------- diff --git a/tests/test_incremental_equivalence.py b/tests/test_incremental_equivalence.py new file mode 100644 index 00000000..493b9813 --- /dev/null +++ b/tests/test_incremental_equivalence.py @@ -0,0 +1,156 @@ +"""PR-T1: Foundation tests — determinism, .deps.json read/write/validate.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from build_ast_graph import ( + ONTOLOGY_VERSION, + FileDeps, + _read_dependency_index, + pass1_parse, + pass2_edges, + pass3_calls, + pass4_routes, + pass5_imperative_edges, + pass6_match_edges, + write_kuzu, +) + +CORPUS = Path(__file__).resolve().parent / "bank-chat-system" + + +def _full_rebuild_into(corpus: Path, db_path: Path) -> Path: + """Run pass1–6 + write_kuzu into db_path; return db_path.""" + from build_ast_graph import GraphTables + + tables = GraphTables() + asts = pass1_parse(corpus, tables, verbose=False) + pass2_edges(tables, asts, verbose=False) + pass3_calls(tables, asts, verbose=False) + pass4_routes(tables, asts, source_root=corpus, verbose=False) + pass5_imperative_edges(tables, asts, source_root=corpus, verbose=False) + pass6_match_edges(tables, verbose=False) + write_kuzu(db_path, tables, source_root=corpus, verbose=False) + return db_path + + +def _dump_node_ids(db_path: Path) -> set[str]: + """Return all Symbol node IDs from a Kuzu database.""" + import kuzu + + db = kuzu.Database(str(db_path)) + conn = kuzu.Connection(db) + result = conn.execute("MATCH (s:Symbol) RETURN s.id AS id") + ids = set() + while result.has_next(): + ids.add(result.get_next()[0]) + conn.close() + return ids + + +def _dump_edge_rows(db_path: Path) -> set[tuple[str, ...]]: + """Return edge tuples (src, dst, label) from all relationship tables.""" + import kuzu + + db = kuzu.Database(str(db_path)) + conn = kuzu.Connection(db) + labels = [ + "DECLARES", "EXTENDS", "IMPLEMENTS", "INJECTS", + "CALLS", "OVERRIDES", "EXPOSES", + "DECLARES_CLIENT", "DECLARES_PRODUCER", + "HTTP_CALLS", "ASYNC_CALLS", + ] + rows: set[tuple[str, ...]] = set() + for label in labels: + try: + result = conn.execute( + f"MATCH (a)-[e:{label}]->(b) RETURN a.id AS src, b.id AS dst" + ) + except Exception: + continue + while result.has_next(): + row = result.get_next() + rows.add((row[0], row[1], label)) + conn.close() + return rows + + +# ---- PR-T1 tests ---- + + +def test_full_rebuild_is_deterministic(tmp_path: Path) -> None: + """Two full rebuilds on the same corpus produce identical graph state.""" + db_a = tmp_path / "a" / "code_graph.kuzu" + db_b = tmp_path / "b" / "code_graph.kuzu" + _full_rebuild_into(CORPUS, db_a) + _full_rebuild_into(CORPUS, db_b) + + nodes_a = _dump_node_ids(db_a) + nodes_b = _dump_node_ids(db_b) + assert nodes_a == nodes_b, ( + f"Node ID sets differ: {len(nodes_a)} vs {len(nodes_b)}" + ) + + edges_a = _dump_edge_rows(db_a) + edges_b = _dump_edge_rows(db_b) + assert edges_a == edges_b, ( + f"Edge sets differ: {len(edges_a)} vs {len(edges_b)}" + ) + + +def test_deps_json_written_on_full_rebuild(tmp_path: Path) -> None: + """After a full rebuild, .deps.json exists and is well-formed.""" + db_path = tmp_path / "code_graph.kuzu" + _full_rebuild_into(CORPUS, db_path) + + deps_path = db_path.parent / ".deps.json" + assert deps_path.is_file(), ".deps.json missing after full rebuild" + + data = json.loads(deps_path.read_text()) + assert data["version"] == 1 + assert data["ontology_version"] == ONTOLOGY_VERSION + assert isinstance(data["files"], dict) + assert len(data["files"]) > 0 + + +def test_deps_json_fields_coverage(tmp_path: Path) -> None: + """Spot-check a known file has expected dependency entries.""" + db_path = tmp_path / "code_graph.kuzu" + _full_rebuild_into(CORPUS, db_path) + deps_path = db_path.parent / ".deps.json" + + idx = _read_dependency_index(deps_path) + assert idx is not None + + # Find ChatIngressController.java — it should have declares, uses_anno, etc. + ctrl_files = [fp for fp in idx.files if "ChatIngressController" in fp] + assert len(ctrl_files) >= 1, "ChatIngressController.java not found in deps index" + + fp = ctrl_files[0] + deps = idx.files[fp] + assert isinstance(deps, FileDeps) + assert deps.ext_hash, "ext_hash should be non-empty" + assert len(deps.declares) >= 1, "ChatIngressController should declare at least one type" + assert any( + "Controller" in a or "Mapping" in a + for a in deps.uses_anno + ), f"Expected controller/mapping annotations, got {deps.uses_anno}" + + +def test_deps_json_stale_detection(tmp_path: Path) -> None: + """_read_dependency_index returns None for stale ontology version.""" + deps_path = tmp_path / ".deps.json" + stale = { + "version": 1, + "ontology_version": 0, # intentionally wrong + "files": {}, + } + deps_path.write_text(json.dumps(stale)) + assert _read_dependency_index(deps_path) is None + + +def test_deps_json_missing_returns_none(tmp_path: Path) -> None: + """_read_dependency_index returns None when file doesn't exist.""" + assert _read_dependency_index(tmp_path / "nonexistent.json") is None