From 9ff7899b757f262af22f1a24b4545ef72dafc7cc Mon Sep 17 00:00:00 2001 From: "Aaron K. Clark" Date: Sun, 2 Aug 2026 05:47:05 -0500 Subject: [PATCH] perf(retrieval): cache the search weighting maps per index generation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `_owners`, `_weight_generated`, and `_weight_superseded` each rebuilt their maps from a full scan of `chunks`/`notes` on every call, so query latency grew with vault size rather than with the fused candidate set — three full scans per query on top of the fusion work. They now share one `_Weights` snapshot built once per index `generation` and cached per process, the same invalidation contract `_vector_matrix` already used: a local refresh clears it, and a write from another process bumps the generation so the next query rebuilds. Behaviour is unchanged; the superseded alias resolution and the generated-note classification are byte-for-byte the same rules, just computed once. Closes #186. Co-Authored-By: Claude Opus 5 --- BACKLOG.md | 8 +-- CHANGELOG.md | 12 +++- pyproject.toml | 2 +- src/omind/__init__.py | 2 +- src/omind/searchindex.py | 120 ++++++++++++++++++++++++-------------- tests/test_searchindex.py | 18 ++++++ uv.lock | 2 +- 7 files changed, 113 insertions(+), 51 deletions(-) diff --git a/BACKLOG.md b/BACKLOG.md index 0922e64..2993095 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -14,10 +14,10 @@ shell hooks, web frontend). The codebase held up unusually well — no correctne bugs found. What follows are the five findings worth tracking; all are perf, test-coverage, hardening, or docs, not defects._ -- [ ] **Per-query full-table Python scans in the search weighting passes** ([#186](https://github.com/CryptoJones/omind/issues/186)) — _perf (retrieval)_ — - `_weight_generated` / `_weight_superseded` / `_owners` re-scan whole tables and rebuild - their maps on every query; key them off the index `generation` like the cached vector - matrix. Latency grows with vault size, not with the fused candidate set. +- [x] **Per-query full-table Python scans in the search weighting passes** ([#186](https://github.com/CryptoJones/omind/issues/186)) — _perf (retrieval)_ — + `_weight_generated`, `_weight_superseded`, and `_owners` now share one map built once + per index `generation` and cached per process, like the packed vector matrix. Query + cost tracks the fused candidate set instead of the vault size. - [ ] **compliance.py recidivism helpers re-parse the whole append-only log per call** ([#188](https://github.com/CryptoJones/omind/issues/188)) — _perf (enforcement)_ — `summary()` parses twice; `learn.escalate()` runs it N+1 times. mtime/size-keyed memo + a size-based rotation (the log has none, unlike hook-failures.log). diff --git a/CHANGELOG.md b/CHANGELOG.md index bed9f16..9a31864 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,17 @@ All notable changes to this project are documented here. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [6.0.0] - 2026-08-02 + +### Performance +- **The search weighting passes no longer re-scan the whole index on every + query** ([#186](https://github.com/CryptoJones/omind/issues/186)). `_owners`, + `_weight_generated`, and `_weight_superseded` each rebuilt their maps from a + full `chunks`/`notes` scan per call, so query latency grew with vault size + rather than with the fused candidate set. They now share one map built once + per index `generation` and cached per process — the same invalidation contract + the packed vector matrix already used, so an external write still takes effect + on the next refresh. ### Changed - **Adopt MCP revision `2026-07-28` (the stateless revision) by moving to the diff --git a/pyproject.toml b/pyproject.toml index a4f914d..da22e8f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "omind" -version = "5.0.1" +version = "6.0.0" description = "Reproduce the OMI/Obsidian memory integration for AI agents, plus a local web app to view, edit, and add memory entries." readme = "README.md" requires-python = ">=3.10" diff --git a/src/omind/__init__.py b/src/omind/__init__.py index 9e5c135..e4c40df 100644 --- a/src/omind/__init__.py +++ b/src/omind/__init__.py @@ -2,4 +2,4 @@ # Copyright 2026 Aaron K. Clark """omind — OMI/Obsidian memory tooling for AI agents.""" -__version__ = "5.0.1" +__version__ = "6.0.0" diff --git a/src/omind/searchindex.py b/src/omind/searchindex.py index f278300..1ced40b 100644 --- a/src/omind/searchindex.py +++ b/src/omind/searchindex.py @@ -176,6 +176,20 @@ class _Chunk: end_line: int +@dataclass(frozen=True) +class _Weights: + """Per-generation derived maps the weighting passes need on every query. + + Building these costs a full scan of ``chunks`` (and ``notes``), so they are + computed once per index generation instead of once per query — the same + caching contract the packed vector matrix uses. + """ + + owners: dict[int, str] + generated: frozenset[int] + superseded: frozenset[str] + + @dataclass class _NoteRow: filename: str @@ -461,6 +475,8 @@ def __init__(self, omi_dir: Path | str, *, model: str | None = None) -> None: # Per-process cache of the packed vector matrix, keyed by the index # generation so a refresh (here or in another process) invalidates it. self._matrix: tuple[str, list[int], Any, Any] | None = None + # Same contract for the chunk-owner map and the weighting-pass sets. + self._weights: tuple[str, _Weights] | None = None self._last_refresh_at = 0.0 self._last_signature: tuple[int, int] | None = None # One connection shared across threads (the web app serves requests on a @@ -617,6 +633,7 @@ def refresh(self, *, vectors: bool = True) -> Refresh | None: db.execute("COMMIT") stats.seconds = time.perf_counter() - started self._matrix = None + self._weights = None self._last_signature = self._cheap_signature() self._last_refresh_at = time.monotonic() return stats @@ -974,8 +991,60 @@ def _rerank( return fused def _owners(self, db: sqlite3.Connection) -> dict[int, str]: - rows = db.execute("SELECT id, filename FROM chunks") - return {int(r["id"]): str(r["filename"]) for r in rows} + return self._weighting(db).owners + + def _weighting(self, db: sqlite3.Connection) -> _Weights: + """Chunk owners plus the generated/superseded sets, cached per generation. + + One pass over ``chunks``/``notes`` serves the vector leg's owner filter + and both weighting passes, instead of three full scans per query. + """ + generation = self._meta(db, "generation") + if self._weights is not None and self._weights[0] == generation: + return self._weights[1] + owners: dict[int, str] = {} + generated: set[int] = set() + aliases: dict[str, str] = {} + superseded: set[str] = set() + note_rows = list( + db.execute("SELECT filename, title, okf_type, supersedes, superseded_by FROM notes") + ) + generated_notes: set[str] = set() + for row in note_rows: + filename = str(row["filename"]) + aliases[filename.lower()] = filename + aliases[Path(filename).stem.lower()] = filename + aliases[str(row["title"]).strip().lower()] = filename + if str(row["superseded_by"]).strip(): + superseded.add(filename) + name = Path(filename).stem.lower() + if ( + str(row["okf_type"]).strip().lower() + in {"journal", "worklog", "checkpoint", "rollup"} + or name.startswith("session journal") + or name.startswith("worklog ") + ): + generated_notes.add(filename) + for row in note_rows: + target = str(row["supersedes"]).strip() + if not target: + continue + clean = target.strip("[]").split("|", 1)[0].split("#", 1)[0].strip().lower() + if resolved := aliases.get(clean): + superseded.add(resolved) + for row in db.execute("SELECT id, filename FROM chunks"): + chunk_id = int(row["id"]) + filename = str(row["filename"]) + owners[chunk_id] = filename + if filename in generated_notes: + generated.add(chunk_id) + weights = _Weights( + owners=owners, + generated=frozenset(generated), + superseded=frozenset(superseded), + ) + self._weights = (generation, weights) + return weights def _recency_leg( self, db: sqlite3.Connection, allowed: set[str] | None, depth: int @@ -990,59 +1059,24 @@ def _recency_leg( int(r["id"]) for r in rows if allowed is None or str(r["filename"]) in allowed ][:depth] - @staticmethod def _weight_generated( - db: sqlite3.Connection, fused: list[tuple[int, float]] + self, db: sqlite3.Connection, fused: list[tuple[int, float]] ) -> list[tuple[int, float]]: """De-prioritise broad machine-written journals without excluding them.""" - rows = db.execute( - "SELECT c.id, c.filename, n.okf_type FROM chunks c" - " JOIN notes n ON n.filename = c.filename" - ) - generated: set[int] = set() - for row in rows: - name = Path(str(row["filename"])).stem.lower() - okf_type = str(row["okf_type"]).strip().lower() - if ( - okf_type in {"journal", "worklog", "checkpoint", "rollup"} - or name.startswith("session journal") - or name.startswith("worklog ") - ): - generated.add(int(row["id"])) + generated = self._weighting(db).generated weighted = [ (chunk_id, score * _GENERATED_WEIGHT if chunk_id in generated else score) for chunk_id, score in fused ] return sorted(weighted, key=lambda item: (-item[1], item[0])) - @staticmethod def _weight_superseded( - db: sqlite3.Connection, fused: list[tuple[int, float]] + self, db: sqlite3.Connection, fused: list[tuple[int, float]] ) -> list[tuple[int, float]]: """De-rank invalidated facts while preserving their searchable history.""" - notes = list( - db.execute("SELECT filename, title, supersedes, superseded_by FROM notes") - ) - aliases: dict[str, str] = {} - superseded: set[str] = set() - for row in notes: - filename = str(row["filename"]) - aliases[filename.lower()] = filename - aliases[Path(filename).stem.lower()] = filename - aliases[str(row["title"]).strip().lower()] = filename - if str(row["superseded_by"]).strip(): - superseded.add(filename) - for row in notes: - target = str(row["supersedes"]).strip() - if not target: - continue - clean = target.strip("[]").split("|", 1)[0].split("#", 1)[0].strip().lower() - if resolved := aliases.get(clean): - superseded.add(resolved) - chunk_owner = { - int(row["id"]): str(row["filename"]) - for row in db.execute("SELECT id, filename FROM chunks") - } + weights = self._weighting(db) + superseded = weights.superseded + chunk_owner = weights.owners weighted = [ ( chunk_id, diff --git a/tests/test_searchindex.py b/tests/test_searchindex.py index 832620f..c6c4f1a 100644 --- a/tests/test_searchindex.py +++ b/tests/test_searchindex.py @@ -263,6 +263,24 @@ def test_superseded_notes_remain_searchable_but_rank_lower(omi: Path) -> None: assert hits[0].filename == "Release v2.md" +def test_weighting_maps_are_built_once_per_generation(omi: Path) -> None: + _note(omi, "Handbook", "curated operations", ["ops"], details="zebracorn rollback") + idx = searchindex.SearchIndex(omi) + idx.search("zebracorn") + first = idx._weights + assert first is not None + idx.search("rollback") + assert idx._weights is first # a second query re-scans nothing + + _note(omi, "Worklog 2026-07-28", "automatic", ["worklog"], details="zebracorn rollback") + idx.refresh() + idx.search("zebracorn") + second = idx._weights + assert second is not None and second is not first # new generation, rebuilt + assert "Worklog 2026-07-28.md" in set(second[1].owners.values()) + assert second[1].generated # the worklog's chunks are marked generated + + def test_query_punctuation_cannot_break_the_match_expression(omi: Path) -> None: """FTS5 operators in user text are data, not syntax (every term is quoted).""" _note(omi, "Quoted", "handling NEAR and OR in queries", ["x"]) diff --git a/uv.lock b/uv.lock index 7193a73..486b28c 100644 --- a/uv.lock +++ b/uv.lock @@ -2354,7 +2354,7 @@ wheels = [ [[package]] name = "omind" -version = "5.0.1" +version = "6.0.0" source = { editable = "." } dependencies = [ { name = "cryptography" },