perf(retrieval): cache the search weighting maps per index generation (v6.0.0)

BACKLOG.md

@@ -14,10 +14,10 @@ shell hooks, web frontend). The codebase held up unusually well — no correctne
 bugs found. What follows are the five findings worth tracking; all are perf, test-coverage,
 hardening, or docs, not defects._
 
-- [ ] **Per-query full-table Python scans in the search weighting passes** ([#186](https://github.com/CryptoJones/omind/issues/186)) — _perf (retrieval)_ —
-  `_weight_generated` / `_weight_superseded` / `_owners` re-scan whole tables and rebuild
-  their maps on every query; key them off the index `generation` like the cached vector
-  matrix. Latency grows with vault size, not with the fused candidate set.
+- [x] **Per-query full-table Python scans in the search weighting passes** ([#186](https://github.com/CryptoJones/omind/issues/186)) — _perf (retrieval)_ —
+  `_weight_generated`, `_weight_superseded`, and `_owners` now share one map built once
+  per index `generation` and cached per process, like the packed vector matrix. Query
+  cost tracks the fused candidate set instead of the vault size.
 - [ ] **compliance.py recidivism helpers re-parse the whole append-only log per call** ([#188](https://github.com/CryptoJones/omind/issues/188)) — _perf (enforcement)_ —
   `summary()` parses twice; `learn.escalate()` runs it N+1 times. mtime/size-keyed memo +
   a size-based rotation (the log has none, unlike hook-failures.log).

CHANGELOG.md

@@ -5,7 +5,17 @@ All notable changes to this project are documented here.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased]
+## [6.0.0] - 2026-08-02
+
+### Performance
+- **The search weighting passes no longer re-scan the whole index on every
+  query** ([#186](https://github.com/CryptoJones/omind/issues/186)). `_owners`,
+  `_weight_generated`, and `_weight_superseded` each rebuilt their maps from a
+  full `chunks`/`notes` scan per call, so query latency grew with vault size
+  rather than with the fused candidate set. They now share one map built once
+  per index `generation` and cached per process — the same invalidation contract
+  the packed vector matrix already used, so an external write still takes effect
+  on the next refresh.
 
 ### Changed
 - **Adopt MCP revision `2026-07-28` (the stateless revision) by moving to the

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "omind"
-version = "5.0.1"
+version = "6.0.0"
 description = "Reproduce the OMI/Obsidian memory integration for AI agents, plus a local web app to view, edit, and add memory entries."
 readme = "README.md"
 requires-python = ">=3.10"

src/omind/__init__.py

@@ -2,4 +2,4 @@
 # Copyright 2026 Aaron K. Clark
 """omind — OMI/Obsidian memory tooling for AI agents."""
 
-__version__ = "5.0.1"
+__version__ = "6.0.0"

src/omind/searchindex.py

@@ -176,6 +176,20 @@ class _Chunk:
     end_line: int
 
 
+@dataclass(frozen=True)
+class _Weights:
+    """Per-generation derived maps the weighting passes need on every query.
+
+    Building these costs a full scan of ``chunks`` (and ``notes``), so they are
+    computed once per index generation instead of once per query — the same
+    caching contract the packed vector matrix uses.
+    """
+
+    owners: dict[int, str]
+    generated: frozenset[int]
+    superseded: frozenset[str]
+
+
 @dataclass
 class _NoteRow:
     filename: str
@@ -461,6 +475,8 @@ def __init__(self, omi_dir: Path | str, *, model: str | None = None) -> None:
         # Per-process cache of the packed vector matrix, keyed by the index
         # generation so a refresh (here or in another process) invalidates it.
         self._matrix: tuple[str, list[int], Any, Any] | None = None
+        # Same contract for the chunk-owner map and the weighting-pass sets.
+        self._weights: tuple[str, _Weights] | None = None
         self._last_refresh_at = 0.0
         self._last_signature: tuple[int, int] | None = None
         # One connection shared across threads (the web app serves requests on a
@@ -617,6 +633,7 @@ def refresh(self, *, vectors: bool = True) -> Refresh | None:
             db.execute("COMMIT")
             stats.seconds = time.perf_counter() - started
             self._matrix = None
+            self._weights = None
             self._last_signature = self._cheap_signature()
             self._last_refresh_at = time.monotonic()
             return stats
@@ -974,8 +991,60 @@ def _rerank(
             return fused
 
     def _owners(self, db: sqlite3.Connection) -> dict[int, str]:
-        rows = db.execute("SELECT id, filename FROM chunks")
-        return {int(r["id"]): str(r["filename"]) for r in rows}
+        return self._weighting(db).owners
+
+    def _weighting(self, db: sqlite3.Connection) -> _Weights:
+        """Chunk owners plus the generated/superseded sets, cached per generation.
+
+        One pass over ``chunks``/``notes`` serves the vector leg's owner filter
+        and both weighting passes, instead of three full scans per query.
+        """
+        generation = self._meta(db, "generation")
+        if self._weights is not None and self._weights[0] == generation:
+            return self._weights[1]
+        owners: dict[int, str] = {}
+        generated: set[int] = set()
+        aliases: dict[str, str] = {}
+        superseded: set[str] = set()
+        note_rows = list(
+            db.execute("SELECT filename, title, okf_type, supersedes, superseded_by FROM notes")
+        )
+        generated_notes: set[str] = set()
+        for row in note_rows:
+            filename = str(row["filename"])
+            aliases[filename.lower()] = filename
+            aliases[Path(filename).stem.lower()] = filename
+            aliases[str(row["title"]).strip().lower()] = filename
+            if str(row["superseded_by"]).strip():
+                superseded.add(filename)
+            name = Path(filename).stem.lower()
+            if (
+                str(row["okf_type"]).strip().lower()
+                in {"journal", "worklog", "checkpoint", "rollup"}
+                or name.startswith("session journal")
+                or name.startswith("worklog ")
+            ):
+                generated_notes.add(filename)
+        for row in note_rows:
+            target = str(row["supersedes"]).strip()
+            if not target:
+                continue
+            clean = target.strip("[]").split("|", 1)[0].split("#", 1)[0].strip().lower()
+            if resolved := aliases.get(clean):
+                superseded.add(resolved)
+        for row in db.execute("SELECT id, filename FROM chunks"):
+            chunk_id = int(row["id"])
+            filename = str(row["filename"])
+            owners[chunk_id] = filename
+            if filename in generated_notes:
+                generated.add(chunk_id)
+        weights = _Weights(
+            owners=owners,
+            generated=frozenset(generated),
+            superseded=frozenset(superseded),
+        )
+        self._weights = (generation, weights)
+        return weights
 
     def _recency_leg(
         self, db: sqlite3.Connection, allowed: set[str] | None, depth: int
@@ -990,59 +1059,24 @@ def _recency_leg(
             int(r["id"]) for r in rows if allowed is None or str(r["filename"]) in allowed
         ][:depth]
 
-    @staticmethod
     def _weight_generated(
-        db: sqlite3.Connection, fused: list[tuple[int, float]]
+        self, db: sqlite3.Connection, fused: list[tuple[int, float]]
     ) -> list[tuple[int, float]]:
         """De-prioritise broad machine-written journals without excluding them."""
-        rows = db.execute(
-            "SELECT c.id, c.filename, n.okf_type FROM chunks c"
-            " JOIN notes n ON n.filename = c.filename"
-        )
-        generated: set[int] = set()
-        for row in rows:
-            name = Path(str(row["filename"])).stem.lower()
-            okf_type = str(row["okf_type"]).strip().lower()
-            if (
-                okf_type in {"journal", "worklog", "checkpoint", "rollup"}
-                or name.startswith("session journal")
-                or name.startswith("worklog ")
-            ):
-                generated.add(int(row["id"]))
+        generated = self._weighting(db).generated
         weighted = [
             (chunk_id, score * _GENERATED_WEIGHT if chunk_id in generated else score)
             for chunk_id, score in fused
         ]
         return sorted(weighted, key=lambda item: (-item[1], item[0]))
 
-    @staticmethod
     def _weight_superseded(
-        db: sqlite3.Connection, fused: list[tuple[int, float]]
+        self, db: sqlite3.Connection, fused: list[tuple[int, float]]
     ) -> list[tuple[int, float]]:
         """De-rank invalidated facts while preserving their searchable history."""
-        notes = list(
-            db.execute("SELECT filename, title, supersedes, superseded_by FROM notes")
-        )
-        aliases: dict[str, str] = {}
-        superseded: set[str] = set()
-        for row in notes:
-            filename = str(row["filename"])
-            aliases[filename.lower()] = filename
-            aliases[Path(filename).stem.lower()] = filename
-            aliases[str(row["title"]).strip().lower()] = filename
-            if str(row["superseded_by"]).strip():
-                superseded.add(filename)
-        for row in notes:
-            target = str(row["supersedes"]).strip()
-            if not target:
-                continue
-            clean = target.strip("[]").split("|", 1)[0].split("#", 1)[0].strip().lower()
-            if resolved := aliases.get(clean):
-                superseded.add(resolved)
-        chunk_owner = {
-            int(row["id"]): str(row["filename"])
-            for row in db.execute("SELECT id, filename FROM chunks")
-        }
+        weights = self._weighting(db)
+        superseded = weights.superseded
+        chunk_owner = weights.owners
         weighted = [
             (
                 chunk_id,

tests/test_searchindex.py

@@ -263,6 +263,24 @@ def test_superseded_notes_remain_searchable_but_rank_lower(omi: Path) -> None:
     assert hits[0].filename == "Release v2.md"
 
 
+def test_weighting_maps_are_built_once_per_generation(omi: Path) -> None:
+    _note(omi, "Handbook", "curated operations", ["ops"], details="zebracorn rollback")
+    idx = searchindex.SearchIndex(omi)
+    idx.search("zebracorn")
+    first = idx._weights
+    assert first is not None
+    idx.search("rollback")
+    assert idx._weights is first  # a second query re-scans nothing
+
+    _note(omi, "Worklog 2026-07-28", "automatic", ["worklog"], details="zebracorn rollback")
+    idx.refresh()
+    idx.search("zebracorn")
+    second = idx._weights
+    assert second is not None and second is not first  # new generation, rebuilt
+    assert "Worklog 2026-07-28.md" in set(second[1].owners.values())
+    assert second[1].generated  # the worklog's chunks are marked generated
+
+
 def test_query_punctuation_cannot_break_the_match_expression(omi: Path) -> None:
     """FTS5 operators in user text are data, not syntax (every term is quoted)."""
     _note(omi, "Quoted", "handling NEAR and OR in queries", ["x"])