From 39c3d893f8b71674ab3a8e10df120e789b3bcdf2 Mon Sep 17 00:00:00 2001 From: Aryan Pardeshi Date: Sat, 29 Aug 2026 02:32:09 +0530 Subject: [PATCH 1/7] feat(core): add dependency-free CJK search tokens Signed-off-by: Aryan Pardeshi --- src/basic_memory/repository/search_query.py | 38 +++++++++++++++++++- tests/repository/test_cjk_search_tokens.py | 40 +++++++++++++++++++++ 2 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 tests/repository/test_cjk_search_tokens.py diff --git a/src/basic_memory/repository/search_query.py b/src/basic_memory/repository/search_query.py index d73d70c0c..89ae839da 100644 --- a/src/basic_memory/repository/search_query.py +++ b/src/basic_memory/repository/search_query.py @@ -24,7 +24,7 @@ r"\ud7b0-\ud7ff" # Hangul Jamo Extended-B r"\uf900-\ufaff" # CJK Compatibility Ideographs r"\uff65-\uff9f" # Halfwidth Katakana - r"]" + r"]+" ) RELAXATION_EDGE_PUNCTUATION = "?!.,;:，。！？；：、" # Written inside a word (Persian U+200C, Indic conjuncts) rather than between words. @@ -300,3 +300,39 @@ def relaxed_query_words(search_text: str | None) -> list[str] | None: return None pruned_words = [token for token in tokens if token not in RELAXATION_STOPWORDS] return _emit_relaxation_terms(pruned_words or tokens) or None + + +def contains_cjk(text: str) -> bool: + """Whether text contains any character from the supported CJK ranges.""" + return RELAXATION_CJK_PATTERN.search(text) is not None + + +def cjk_bigram_tokens(run: str) -> tuple[str, ...]: + """Overlapping two-character windows over one CJK run. + + Un-segmented CJK text has no whitespace between words, so finding a + substring match that starts mid-run needs overlapping bigrams rather than + whitespace-delimited tokens. A single character has no second character to + pair with, so it is kept as a one-character token instead of vanishing. + """ + if not run: + return () + if len(run) == 1: + return (run,) + return tuple(run[index : index + 2] for index in range(len(run) - 1)) + + +def cjk_search_tokens(*fields: str | None) -> str: + """Space-joined bigram tokens for every CJK run across the given fields. + + Each field is scanned independently so a run can never cross a field + boundary, and non-CJK substrings (Latin words, digits) never enter the + auxiliary token stream. + """ + tokens: list[str] = [] + for field in fields: + if not field: + continue + for match in RELAXATION_CJK_PATTERN.finditer(field): + tokens.extend(cjk_bigram_tokens(match.group(0))) + return " ".join(tokens) diff --git a/tests/repository/test_cjk_search_tokens.py b/tests/repository/test_cjk_search_tokens.py new file mode 100644 index 000000000..e87014c33 --- /dev/null +++ b/tests/repository/test_cjk_search_tokens.py @@ -0,0 +1,40 @@ +"""Pure tokenization primitives for the derived CJK bigram search-token channel.""" + +import pytest + +from basic_memory.repository.search_query import ( + cjk_bigram_tokens, + cjk_search_tokens, + contains_cjk, +) + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("", ()), + ("适", ("适",)), + ("适者生存", ("适者", "者生", "生存")), + ("東京都", ("東京", "京都")), + ("かなカナ", ("かな", "なカ", "カナ")), + ("한국어", ("한국", "국어")), + ], +) +def test_cjk_bigram_tokens_use_overlapping_windows(text: str, expected: tuple[str, ...]) -> None: + assert cjk_bigram_tokens(text) == expected + + +def test_cjk_search_tokens_exclude_non_cjk_and_preserve_run_boundaries() -> None: + assert cjk_search_tokens("iPhone很好用", "计划", "开始") == "很好 好用 计划 开始" + + +def test_cjk_search_tokens_skips_empty_and_none_fields() -> None: + assert cjk_search_tokens("", None, "适者") == "适者" + + +@pytest.mark.parametrize( + ("text", "expected"), + [("plain ASCII", False), ("مرحبا", False), ("适者", True), ("カナ", True), ("한국", True)], +) +def test_contains_cjk_covers_supported_ranges(text: str, expected: bool) -> None: + assert contains_cjk(text) is expected From 2ca134ee260f1e488d98dc4d49b4de97bc5587ea Mon Sep 17 00:00:00 2001 From: Aryan Pardeshi Date: Sat, 29 Aug 2026 02:59:37 +0530 Subject: [PATCH 2/7] feat(core): add CJK search token indexes Signed-off-by: Aryan Pardeshi --- .../8a7b6c5d4e3f_add_cjk_search_tokens.py | 175 ++++++++++++ src/basic_memory/models/search.py | 22 ++ tests/test_cjk_search_tokens_migration.py | 258 ++++++++++++++++++ 3 files changed, 455 insertions(+) create mode 100644 src/basic_memory/alembic/versions/8a7b6c5d4e3f_add_cjk_search_tokens.py create mode 100644 tests/test_cjk_search_tokens_migration.py diff --git a/src/basic_memory/alembic/versions/8a7b6c5d4e3f_add_cjk_search_tokens.py b/src/basic_memory/alembic/versions/8a7b6c5d4e3f_add_cjk_search_tokens.py new file mode 100644 index 000000000..14107d195 --- /dev/null +++ b/src/basic_memory/alembic/versions/8a7b6c5d4e3f_add_cjk_search_tokens.py @@ -0,0 +1,175 @@ +"""Add CJK search token indexes + +Revision ID: 8a7b6c5d4e3f +Revises: 7f6a2b8c9d10 +Create Date: 2026-08-29 23:30:00.000000 + +""" + +from typing import Sequence, Union + +from alembic import op +from sqlalchemy import text + + +# revision identifiers, used by Alembic. +revision: str = "8a7b6c5d4e3f" +down_revision: Union[str, None] = "7f6a2b8c9d10" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def _has_fts5_search_index(connection) -> bool: + """Whether a real FTS5 search_index virtual table exists on this connection. + + Trigger: SQLite creates search_index as an FTS5 virtual table at runtime + via SearchRepository.init_search_index, not through Alembic, so fresh + installs hit this migration before the table exists. Some migration + tests also stand up a minimal plain `search_index` table to exercise an + earlier migration's repair SQL in isolation. + Why: recreating a table Alembic never created would make a fresh install + diverge from every other install; recreating a same-named table that + isn't actually the FTS5 index would destroy unrelated data for no schema + benefit. + Outcome: only a genuine FTS5 search_index gets dropped and rebuilt with + search_tokens. A missing table, or a differently-shaped one, is left + alone -- the runtime creates the real one with the current schema on + first use. + """ + row = connection.execute( + text("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'search_index'") + ).fetchone() + return row is not None and row[0] is not None and "fts5" in row[0].lower() + + +def upgrade() -> None: + """Add derived CJK search-token storage for SQLite and PostgreSQL. + + Schema only: the Python bigram transform (repository/search_query.py + cjk_search_tokens, added in the prior commit) is deliberately not + duplicated in SQL or PL/pgSQL. Existing rows stay stale until the forced + reindex below runs, consistent with this project's derived-state + convergence model. + """ + connection = op.get_bind() + if connection.dialect.name == "sqlite" and _has_fts5_search_index(connection): + # search_index is a derived FTS5 virtual table with no foreign keys to + # entity/note_content/observation/relation (FTS5 can't carry one), so + # dropping and recreating it cannot touch canonical source data -- it + # only empties the derived index, which the reindex below repopulates. + op.execute("DROP TABLE IF EXISTS search_index") + op.execute(""" + CREATE VIRTUAL TABLE IF NOT EXISTS search_index USING fts5( + -- Core entity fields + id UNINDEXED, -- Row ID + title, -- Title for searching + content_stems, -- Main searchable content split into stems + content_snippet, -- File content snippet for display + search_tokens, -- Derived CJK bigram tokens (cjk_search_tokens) + permalink, -- Stable identifier (now indexed for path search) + file_path UNINDEXED, -- Physical location + type UNINDEXED, -- entity/relation/observation + + -- Project context + project_id UNINDEXED, -- Project identifier + + -- Relation fields + from_id UNINDEXED, -- Source entity + to_id UNINDEXED, -- Target entity + relation_type UNINDEXED, -- Type of relation + + -- Observation fields + entity_id UNINDEXED, -- Parent entity + category UNINDEXED, -- Observation category + + -- Common fields + metadata UNINDEXED, -- JSON metadata + created_at UNINDEXED, -- Creation timestamp + updated_at UNINDEXED, -- Last update + + -- Configuration + tokenize='unicode61 tokenchars 0x2F', -- Hex code for / + prefix='1,2,3,4' -- Support longer prefixes for paths + ); + """) + elif connection.dialect.name == "postgresql": + op.execute("ALTER TABLE search_index ADD COLUMN IF NOT EXISTS search_tokens TEXT") + op.execute(""" + ALTER TABLE search_index ADD COLUMN IF NOT EXISTS search_tokens_index_col tsvector + GENERATED ALWAYS AS ( + to_tsvector('simple', coalesce(search_tokens, '')) + ) STORED + """) + op.execute(""" + CREATE INDEX IF NOT EXISTS idx_search_index_cjk_fts + ON search_index USING gin(search_tokens_index_col) + """) + + op.execute("ALTER TABLE search_index_fts_chunks ADD COLUMN IF NOT EXISTS chunk_tokens TEXT") + op.execute(""" + ALTER TABLE search_index_fts_chunks + ADD COLUMN IF NOT EXISTS chunk_tokens_index_col tsvector + GENERATED ALWAYS AS ( + to_tsvector('simple', coalesce(chunk_tokens, '')) + ) STORED + """) + op.execute(""" + CREATE INDEX IF NOT EXISTS idx_search_index_fts_chunks_cjk_fts + ON search_index_fts_chunks USING gin(chunk_tokens_index_col) + """) + + print("\nCJK search index added. Run: basic-memory reindex --full --search\n") + + +def downgrade() -> None: + """Remove the CJK search-token storage, restoring the prior schema exactly.""" + connection = op.get_bind() + if connection.dialect.name == "sqlite" and _has_fts5_search_index(connection): + op.execute("DROP TABLE IF EXISTS search_index") + op.execute(""" + CREATE VIRTUAL TABLE IF NOT EXISTS search_index USING fts5( + -- Core entity fields + id UNINDEXED, -- Row ID + title, -- Title for searching + content_stems, -- Main searchable content split into stems + content_snippet, -- File content snippet for display + permalink, -- Stable identifier (now indexed for path search) + file_path UNINDEXED, -- Physical location + type UNINDEXED, -- entity/relation/observation + + -- Project context + project_id UNINDEXED, -- Project identifier + + -- Relation fields + from_id UNINDEXED, -- Source entity + to_id UNINDEXED, -- Target entity + relation_type UNINDEXED, -- Type of relation + + -- Observation fields + entity_id UNINDEXED, -- Parent entity + category UNINDEXED, -- Observation category + + -- Common fields + metadata UNINDEXED, -- JSON metadata + created_at UNINDEXED, -- Creation timestamp + updated_at UNINDEXED, -- Last update + + -- Configuration + tokenize='unicode61 tokenchars 0x2F', -- Hex code for / + prefix='1,2,3,4' -- Support longer prefixes for paths + ); + """) + elif connection.dialect.name == "postgresql": + op.execute("DROP INDEX IF EXISTS idx_search_index_fts_chunks_cjk_fts") + op.execute(""" + ALTER TABLE search_index_fts_chunks + DROP COLUMN IF EXISTS chunk_tokens_index_col + """) + op.execute("ALTER TABLE search_index_fts_chunks DROP COLUMN IF EXISTS chunk_tokens") + + op.execute("DROP INDEX IF EXISTS idx_search_index_cjk_fts") + op.execute(""" + ALTER TABLE search_index + DROP COLUMN IF EXISTS search_tokens_index_col + """) + op.execute("ALTER TABLE search_index DROP COLUMN IF EXISTS search_tokens") diff --git a/src/basic_memory/models/search.py b/src/basic_memory/models/search.py index c9616d850..3d2224a96 100644 --- a/src/basic_memory/models/search.py +++ b/src/basic_memory/models/search.py @@ -39,6 +39,10 @@ coalesce(content_stems, '') ) ) STORED, + search_tokens TEXT, + search_tokens_index_col tsvector GENERATED ALWAYS AS ( + to_tsvector('simple', coalesce(search_tokens, '')) + ) STORED, PRIMARY KEY (id, type, project_id), FOREIGN KEY (project_id) REFERENCES project(id) ON DELETE CASCADE ) @@ -48,6 +52,14 @@ CREATE INDEX IF NOT EXISTS idx_search_index_fts ON search_index USING gin(textsearchable_index_col) """) +# Cross-CJK lexical candidates: search_tokens holds whitespace-separated +# overlapping bigrams for CJK runs (see repository/search_query.py +# cjk_search_tokens), indexed with the 'simple' parser so Postgres does not +# stem or stopword-filter the bigrams. +CREATE_POSTGRES_SEARCH_INDEX_CJK_FTS = DDL(""" +CREATE INDEX IF NOT EXISTS idx_search_index_cjk_fts ON search_index USING gin(search_tokens_index_col) +""") + # Full note bodies are stored in bounded child rows so one unusually large note # cannot exceed PostgreSQL's per-tsvector size limit. CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_TABLE = DDL(""" @@ -60,6 +72,10 @@ textsearchable_index_col tsvector GENERATED ALWAYS AS ( to_tsvector('english', chunk_text) ) STORED, + chunk_tokens TEXT, + chunk_tokens_index_col tsvector GENERATED ALWAYS AS ( + to_tsvector('simple', coalesce(chunk_tokens, '')) + ) STORED, PRIMARY KEY (project_id, search_index_id, search_index_type, chunk_index), FOREIGN KEY (search_index_id, search_index_type, project_id) REFERENCES search_index(id, type, project_id) @@ -72,6 +88,11 @@ ON search_index_fts_chunks USING gin(textsearchable_index_col) """) +CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_CJK_FTS = DDL(""" +CREATE INDEX IF NOT EXISTS idx_search_index_fts_chunks_cjk_fts +ON search_index_fts_chunks USING gin(chunk_tokens_index_col) +""") + CREATE_POSTGRES_SEARCH_INDEX_METADATA = DDL(""" CREATE INDEX IF NOT EXISTS idx_search_index_metadata_gin ON search_index USING gin(metadata jsonb_path_ops) """) @@ -94,6 +115,7 @@ title, -- Title for searching content_stems, -- Main searchable content split into stems content_snippet, -- File content snippet for display + search_tokens, -- Derived CJK bigram tokens (cjk_search_tokens) permalink, -- Stable identifier (now indexed for path search) file_path UNINDEXED, -- Physical location type UNINDEXED, -- entity/relation/observation diff --git a/tests/test_cjk_search_tokens_migration.py b/tests/test_cjk_search_tokens_migration.py new file mode 100644 index 000000000..ee8d2ab22 --- /dev/null +++ b/tests/test_cjk_search_tokens_migration.py @@ -0,0 +1,258 @@ +"""Tests for the CJK search token schema migration.""" + +import sqlite3 +from importlib import import_module +from pathlib import Path +from types import SimpleNamespace + +from alembic import command +from alembic.config import Config + +from basic_memory import db + +migration = import_module("basic_memory.alembic.versions.8a7b6c5d4e3f_add_cjk_search_tokens") + + +# Column set on the SQLite FTS5 search_index table immediately before this +# migration (i.e. the schema produced by models/search.py's CREATE_SEARCH_INDEX +# as of revision 7f6a2b8c9d10). Used to assert the migration adds exactly one +# new column and that downgrade restores exactly this set. +PRE_MIGRATION_SEARCH_INDEX_COLUMNS = { + "id", + "title", + "content_stems", + "content_snippet", + "permalink", + "file_path", + "type", + "project_id", + "from_id", + "to_id", + "relation_type", + "entity_id", + "category", + "metadata", + "created_at", + "updated_at", +} + + +def _sqlite_alembic_config(database_path: Path) -> Config: + """Build an Alembic config that upgrades a temporary SQLite database.""" + alembic_dir = Path(db.__file__).parent / "alembic" + config = Config() + config.set_main_option("script_location", str(alembic_dir)) + config.set_main_option("revision_environment", "false") + config.set_main_option("sqlalchemy.url", f"sqlite:///{database_path}") + return config + + +def _connection(dialect_name: str) -> SimpleNamespace: + return SimpleNamespace(dialect=SimpleNamespace(name=dialect_name)) + + +def _table_columns(connection: sqlite3.Connection, table_name: str) -> set[str]: + return {row[1] for row in connection.execute(f"PRAGMA table_info({table_name})").fetchall()} + + +# --- SQLite: real database, real Alembic upgrade/downgrade --- + + +def _seed_legacy_search_index_with_data(connection: sqlite3.Connection) -> None: + """Simulate a database where SearchRepository.init_search_index already + created the runtime FTS5 table (pre-migration shape) with a row in it, + alongside the canonical project/entity rows it was derived from.""" + timestamp = "2026-08-29 00:00:00" + connection.execute( + """ + INSERT INTO project ( + id, name, permalink, path, is_active, is_default, + created_at, updated_at, external_id + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + (1, "test", "test", "/test", True, True, timestamp, timestamp, "project-1"), + ) + connection.execute( + """ + INSERT INTO entity ( + id, title, note_type, content_type, file_path, + created_at, updated_at, project_id, external_id + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + (1, "Source", "note", "text/markdown", "source.md", timestamp, timestamp, 1, "entity-1"), + ) + connection.execute( + "CREATE VIRTUAL TABLE search_index USING fts5(" + "id UNINDEXED, title, content_stems, content_snippet, permalink, " + "file_path UNINDEXED, type UNINDEXED, project_id UNINDEXED, " + "from_id UNINDEXED, to_id UNINDEXED, relation_type UNINDEXED, " + "entity_id UNINDEXED, category UNINDEXED, metadata UNINDEXED, " + "created_at UNINDEXED, updated_at UNINDEXED)" + ) + connection.execute( + "INSERT INTO search_index (id, title, type, project_id) VALUES (1, 'Source', 'entity', 1)" + ) + connection.commit() + + +def test_sqlite_upgrade_skips_missing_search_index(tmp_path, monkeypatch) -> None: + """A fresh install has no runtime FTS5 table yet; the migration must not + create one -- that stays SearchRepository.init_search_index's job, and it + will build the table with search_tokens already present.""" + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("BASIC_MEMORY_HOME", str(tmp_path / "basic-memory")) + database_path = tmp_path / "cjk-search-tokens-fresh.db" + config = _sqlite_alembic_config(database_path) + + command.upgrade(config, "head") + + connection = sqlite3.connect(database_path) + try: + search_index_exists = connection.execute( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'search_index'" + ).fetchone() + finally: + connection.close() + assert search_index_exists is None + + +def test_sqlite_upgrade_adds_search_tokens_and_preserves_source_data(tmp_path, monkeypatch) -> None: + """Recreating an existing derived FTS5 index adds search_tokens without + disturbing canonical source rows.""" + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("BASIC_MEMORY_HOME", str(tmp_path / "basic-memory")) + database_path = tmp_path / "cjk-search-tokens-data.db" + config = _sqlite_alembic_config(database_path) + command.upgrade(config, "7f6a2b8c9d10") + connection = sqlite3.connect(database_path) + try: + _seed_legacy_search_index_with_data(connection) + finally: + connection.close() + + command.upgrade(config, "head") + + connection = sqlite3.connect(database_path) + try: + entity_rows = connection.execute("SELECT id, title FROM entity").fetchall() + project_rows = connection.execute("SELECT id, name FROM project").fetchall() + # The derived index itself is legitimately emptied by the recreate; + # only the canonical entity/project rows must survive untouched. + search_rows = connection.execute("SELECT id FROM search_index").fetchall() + columns = _table_columns(connection, "search_index") + finally: + connection.close() + assert entity_rows == [(1, "Source")] + assert project_rows == [(1, "test")] + assert search_rows == [] + assert columns == PRE_MIGRATION_SEARCH_INDEX_COLUMNS | {"search_tokens"} + + +def test_sqlite_downgrade_removes_search_tokens_column(tmp_path, monkeypatch) -> None: + """Downgrade must restore exactly the pre-migration column set and must + not disturb canonical source rows either.""" + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("BASIC_MEMORY_HOME", str(tmp_path / "basic-memory")) + database_path = tmp_path / "cjk-search-tokens-downgrade.db" + config = _sqlite_alembic_config(database_path) + command.upgrade(config, "7f6a2b8c9d10") + connection = sqlite3.connect(database_path) + try: + _seed_legacy_search_index_with_data(connection) + finally: + connection.close() + command.upgrade(config, "head") + + command.downgrade(config, "7f6a2b8c9d10") + + connection = sqlite3.connect(database_path) + try: + columns = _table_columns(connection, "search_index") + entity_rows = connection.execute("SELECT id, title FROM entity").fetchall() + finally: + connection.close() + assert columns == PRE_MIGRATION_SEARCH_INDEX_COLUMNS + assert "search_tokens" not in columns + assert entity_rows == [(1, "Source")] + + +# --- PostgreSQL: dialect-branch unit tests (no live database required) --- +# +# Mirrors the monkeypatch technique in test_postgres_full_content_search_migration.py, +# the test for this migration's own down_revision (7f6a2b8c9d10): capture the +# literal SQL passed to op.execute rather than asserting against a live +# information_schema/pg_indexes query, since standing up Postgres here would +# require Docker/testcontainers. + + +def _normalized_statements(statements: list[str]) -> list[str]: + """Collapse each captured statement's whitespace so line-wrapping in the + migration source can't change whether an assertion matches.""" + return [" ".join(statement.split()) for statement in statements] + + +def test_postgres_upgrade_adds_token_columns_vectors_and_gin_indexes(monkeypatch) -> None: + statements: list[str] = [] + monkeypatch.setattr(migration.op, "get_bind", lambda: _connection("postgresql")) + monkeypatch.setattr( + migration.op, "execute", lambda statement: statements.append(str(statement)) + ) + + migration.upgrade() + + normalized = _normalized_statements(statements) + assert "ALTER TABLE search_index ADD COLUMN IF NOT EXISTS search_tokens TEXT" in normalized + assert any( + "search_tokens_index_col tsvector GENERATED ALWAYS AS" in statement + and "to_tsvector('simple', coalesce(search_tokens, ''))" in statement + and "STORED" in statement + for statement in normalized + ) + assert ( + "CREATE INDEX IF NOT EXISTS idx_search_index_cjk_fts " + "ON search_index USING gin(search_tokens_index_col)" in normalized + ) + + assert ( + "ALTER TABLE search_index_fts_chunks ADD COLUMN IF NOT EXISTS chunk_tokens TEXT" + in normalized + ) + assert any( + "chunk_tokens_index_col tsvector GENERATED ALWAYS AS" in statement + and "to_tsvector('simple', coalesce(chunk_tokens, ''))" in statement + and "STORED" in statement + for statement in normalized + ) + assert ( + "CREATE INDEX IF NOT EXISTS idx_search_index_fts_chunks_cjk_fts " + "ON search_index_fts_chunks USING gin(chunk_tokens_index_col)" in normalized + ) + + # Exactly two columns + one index per table (row-level, then chunk-level); + # nothing else runs on the postgresql branch. + assert len(normalized) == 6 + + +def test_postgres_downgrade_removes_only_new_columns_and_indexes(monkeypatch) -> None: + statements: list[str] = [] + monkeypatch.setattr(migration.op, "get_bind", lambda: _connection("postgresql")) + monkeypatch.setattr( + migration.op, "execute", lambda statement: statements.append(str(statement)) + ) + + migration.downgrade() + + normalized = _normalized_statements(statements) + assert "DROP INDEX IF EXISTS idx_search_index_cjk_fts" in normalized + assert "DROP INDEX IF EXISTS idx_search_index_fts_chunks_cjk_fts" in normalized + assert "ALTER TABLE search_index DROP COLUMN IF EXISTS search_tokens_index_col" in normalized + assert "ALTER TABLE search_index DROP COLUMN IF EXISTS search_tokens" in normalized + assert ( + "ALTER TABLE search_index_fts_chunks DROP COLUMN IF EXISTS chunk_tokens_index_col" + in normalized + ) + assert "ALTER TABLE search_index_fts_chunks DROP COLUMN IF EXISTS chunk_tokens" in normalized + + # Exactly the two new indexes and four new columns come off; a stray + # seventh statement would mean a pre-existing column/index got touched. + assert len(normalized) == 6 From 1cf9215642e5d9e4af98d64e8b32ae720181c7b3 Mon Sep 17 00:00:00 2001 From: Aryan Pardeshi Date: Sat, 29 Aug 2026 03:15:21 +0530 Subject: [PATCH 3/7] test(core): create CJK indexes in PostgreSQL fixtures Signed-off-by: Aryan Pardeshi --- .../8a7b6c5d4e3f_add_cjk_search_tokens.py | 3 +- tests/conftest.py | 4 ++ tests/test_cjk_search_tokens_migration.py | 56 +++++++++++++++++++ 3 files changed, 62 insertions(+), 1 deletion(-) diff --git a/src/basic_memory/alembic/versions/8a7b6c5d4e3f_add_cjk_search_tokens.py b/src/basic_memory/alembic/versions/8a7b6c5d4e3f_add_cjk_search_tokens.py index 14107d195..3e3ee8b88 100644 --- a/src/basic_memory/alembic/versions/8a7b6c5d4e3f_add_cjk_search_tokens.py +++ b/src/basic_memory/alembic/versions/8a7b6c5d4e3f_add_cjk_search_tokens.py @@ -10,6 +10,7 @@ from alembic import op from sqlalchemy import text +from sqlalchemy.engine import Connection # revision identifiers, used by Alembic. @@ -19,7 +20,7 @@ depends_on: Union[str, Sequence[str], None] = None -def _has_fts5_search_index(connection) -> bool: +def _has_fts5_search_index(connection: Connection) -> bool: """Whether a real FTS5 search_index virtual table exists on this connection. Trigger: SQLite creates search_index as an FTS5 virtual table at runtime diff --git a/tests/conftest.py b/tests/conftest.py index ef6cdb9d1..6657739a1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -151,7 +151,9 @@ def _resolve_postgres_sync_url(postgres_container) -> str: async def _reset_postgres_test_schema(engine: AsyncEngine, async_url: str) -> None: """Restore the shared Postgres schema to a clean baseline.""" from basic_memory.models.search import ( + CREATE_POSTGRES_SEARCH_INDEX_CJK_FTS, CREATE_POSTGRES_SEARCH_INDEX_FTS, + CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_CJK_FTS, CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_INDEX, CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_TABLE, CREATE_POSTGRES_SEARCH_INDEX_METADATA, @@ -168,8 +170,10 @@ async def _reset_postgres_test_schema(engine: AsyncEngine, async_url: str) -> No await conn.run_sync(Base.metadata.create_all) await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_TABLE) await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_FTS) + await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_CJK_FTS) await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_TABLE) await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_INDEX) + await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_CJK_FTS) await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_METADATA) await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_PERMALINK) await conn.execute(CREATE_POSTGRES_SEARCH_VECTOR_CHUNKS_TABLE) diff --git a/tests/test_cjk_search_tokens_migration.py b/tests/test_cjk_search_tokens_migration.py index ee8d2ab22..95c9a9b6c 100644 --- a/tests/test_cjk_search_tokens_migration.py +++ b/tests/test_cjk_search_tokens_migration.py @@ -1,14 +1,18 @@ """Tests for the CJK search token schema migration.""" +from collections.abc import Callable import sqlite3 from importlib import import_module from pathlib import Path from types import SimpleNamespace +from typing import cast from alembic import command from alembic.config import Config +import pytest from basic_memory import db +from tests import conftest as test_conftest migration = import_module("basic_memory.alembic.versions.8a7b6c5d4e3f_add_cjk_search_tokens") @@ -51,6 +55,42 @@ def _connection(dialect_name: str) -> SimpleNamespace: return SimpleNamespace(dialect=SimpleNamespace(name=dialect_name)) +class _FakeResult: + def scalar(self) -> None: + return None + + +class _FakeConnection: + def __init__(self) -> None: + self.statements: list[object] = [] + + async def run_sync(self, callback: Callable[..., object]) -> None: + del callback + + async def execute(self, statement: object) -> _FakeResult: + self.statements.append(statement) + return _FakeResult() + + +class _FakeTransaction: + def __init__(self, connection: _FakeConnection) -> None: + self.connection = connection + + async def __aenter__(self) -> _FakeConnection: + return self.connection + + async def __aexit__(self, exc_type: object, exc: object, traceback: object) -> None: + del exc_type, exc, traceback + + +class _FakeEngine: + def __init__(self, connection: _FakeConnection) -> None: + self.connection = connection + + def begin(self) -> _FakeTransaction: + return _FakeTransaction(self.connection) + + def _table_columns(connection: sqlite3.Connection, table_name: str) -> set[str]: return {row[1] for row in connection.execute(f"PRAGMA table_info({table_name})").fetchall()} @@ -256,3 +296,19 @@ def test_postgres_downgrade_removes_only_new_columns_and_indexes(monkeypatch) -> # Exactly the two new indexes and four new columns come off; a stray # seventh statement would mean a pre-existing column/index got touched. assert len(normalized) == 6 + + +@pytest.mark.asyncio +async def test_postgres_fixture_recreates_cjk_indexes_without_docker(monkeypatch) -> None: + """The shared Postgres fixture must execute both CJK index DDL statements.""" + connection = _FakeConnection() + monkeypatch.setattr(test_conftest.command, "stamp", lambda *args, **kwargs: None) + + await test_conftest._reset_postgres_test_schema( + cast(test_conftest.AsyncEngine, _FakeEngine(connection)), + "postgresql+asyncpg://test/test", + ) + + normalized = [" ".join(str(statement).split()) for statement in connection.statements] + assert any("idx_search_index_cjk_fts" in statement for statement in normalized) + assert any("idx_search_index_fts_chunks_cjk_fts" in statement for statement in normalized) From 7856444ee85eb3425552b0ec88da92b6f5260470 Mon Sep 17 00:00:00 2001 From: Aryan Pardeshi Date: Sat, 29 Aug 2026 03:33:56 +0530 Subject: [PATCH 4/7] feat(core): populate CJK search token indexes Signed-off-by: Aryan Pardeshi --- .../repository/postgres_search_repository.py | 41 ++++--- .../repository/search_index_row.py | 3 + .../repository/search_repository_base.py | 12 +- src/basic_memory/services/search_service.py | 24 +++- .../test_postgres_search_repository.py | 52 ++++++++ tests/repository/test_search_repository.py | 111 ++++++++++++++++++ tests/services/test_search_service.py | 67 +++++++++++ 7 files changed, 282 insertions(+), 28 deletions(-) diff --git a/src/basic_memory/repository/postgres_search_repository.py b/src/basic_memory/repository/postgres_search_repository.py index d468267c1..e292fd4cd 100644 --- a/src/basic_memory/repository/postgres_search_repository.py +++ b/src/basic_memory/repository/postgres_search_repository.py @@ -20,7 +20,7 @@ from basic_memory.repository.rerank_provider import RerankProvider from basic_memory.repository.rerank_provider_factory import create_rerank_provider from basic_memory.repository.search_index_row import SearchIndexRow -from basic_memory.repository.search_query import relaxed_query_words +from basic_memory.repository.search_query import cjk_search_tokens, relaxed_query_words from basic_memory.repository.semantic_chunking import VectorChunkRecord from basic_memory.repository.search_repository_base import ( SearchRepositoryBase, @@ -229,13 +229,13 @@ async def index_item(self, search_index_row: SearchIndexRow) -> None: await session.execute( text(""" INSERT INTO search_index ( - id, title, content_stems, content_snippet, permalink, file_path, type, metadata, + id, title, content_stems, content_snippet, search_tokens, permalink, file_path, type, metadata, from_id, to_id, relation_type, entity_id, category, created_at, updated_at, project_id ) VALUES ( - :id, :title, :content_stems, :content_snippet, :permalink, :file_path, :type, :metadata, + :id, :title, :content_stems, :content_snippet, :search_tokens, :permalink, :file_path, :type, :metadata, :from_id, :to_id, :relation_type, :entity_id, :category, :created_at, :updated_at, @@ -246,6 +246,7 @@ async def index_item(self, search_index_row: SearchIndexRow) -> None: title = EXCLUDED.title, content_stems = EXCLUDED.content_stems, content_snippet = EXCLUDED.content_snippet, + search_tokens = EXCLUDED.search_tokens, file_path = EXCLUDED.file_path, type = EXCLUDED.type, metadata = EXCLUDED.metadata, @@ -291,16 +292,19 @@ async def _replace_fts_chunks( }, ) - chunks = [ - { - "search_index_id": row.id, - "search_index_type": row.type, - "chunk_index": chunk_index, - "chunk_text": chunk_text.replace("\x00", ""), - } - for row in search_index_rows - for chunk_index, chunk_text in _iter_fts_chunks(row.content_snippet) - ] + chunks = [] + for row in search_index_rows: + for chunk_index, chunk_text in _iter_fts_chunks(row.content_snippet): + chunk_text = chunk_text.replace("\x00", "") + chunks.append( + { + "search_index_id": row.id, + "search_index_type": row.type, + "chunk_index": chunk_index, + "chunk_text": chunk_text, + "chunk_tokens": cjk_search_tokens(chunk_text), + } + ) if not chunks: return @@ -311,7 +315,8 @@ async def _replace_fts_chunks( search_index_id, search_index_type, chunk_index, - chunk_text + chunk_text, + chunk_tokens ) SELECT :project_id, @@ -323,7 +328,8 @@ async def _replace_fts_chunks( search_index_id INTEGER, search_index_type VARCHAR, chunk_index INTEGER, - chunk_text TEXT + chunk_text TEXT, + chunk_tokens TEXT ) """), {"project_id": self.project_id, "chunks": json.dumps(chunks)}, @@ -754,13 +760,13 @@ async def bulk_index_items(self, search_index_rows: List[SearchIndexRow]) -> Non await session.execute( text(""" INSERT INTO search_index ( - id, title, content_stems, content_snippet, permalink, file_path, type, metadata, + id, title, content_stems, content_snippet, search_tokens, permalink, file_path, type, metadata, from_id, to_id, relation_type, entity_id, category, created_at, updated_at, project_id ) VALUES ( - :id, :title, :content_stems, :content_snippet, :permalink, :file_path, :type, :metadata, + :id, :title, :content_stems, :content_snippet, :search_tokens, :permalink, :file_path, :type, :metadata, :from_id, :to_id, :relation_type, :entity_id, :category, :created_at, :updated_at, @@ -771,6 +777,7 @@ async def bulk_index_items(self, search_index_rows: List[SearchIndexRow]) -> Non title = EXCLUDED.title, content_stems = EXCLUDED.content_stems, content_snippet = EXCLUDED.content_snippet, + search_tokens = EXCLUDED.search_tokens, file_path = EXCLUDED.file_path, type = EXCLUDED.type, metadata = EXCLUDED.metadata, diff --git a/src/basic_memory/repository/search_index_row.py b/src/basic_memory/repository/search_index_row.py index 01d63879c..e49f5bb9c 100644 --- a/src/basic_memory/repository/search_index_row.py +++ b/src/basic_memory/repository/search_index_row.py @@ -34,6 +34,7 @@ class SearchIndexRow: title: Optional[str] = None # entity content_stems: Optional[str] = None # entity, observation content_snippet: Optional[str] = None # entity, observation + search_tokens: Optional[str] = None # derived CJK search tokens entity_id: Optional[int] = None # observations category: Optional[str] = None # observations from_id: Optional[int] = None # relations @@ -70,6 +71,7 @@ def from_mapping(cls, row: Mapping[str, Any]) -> "SearchIndexRow": entity_id=row.get("entity_id"), content_stems=row.get("content_stems"), content_snippet=row.get("content_snippet"), + search_tokens=row.get("search_tokens"), category=row.get("category"), created_at=row["created_at"], updated_at=row["updated_at"], @@ -127,6 +129,7 @@ def to_insert(self, serialize_json: bool = True): "title": self.title, "content_stems": self.content_stems, "content_snippet": self.content_snippet, + "search_tokens": self.search_tokens, "permalink": self.permalink, "file_path": self.file_path, "type": self.type, diff --git a/src/basic_memory/repository/search_repository_base.py b/src/basic_memory/repository/search_repository_base.py index 81ed70c9f..5e5bf717a 100644 --- a/src/basic_memory/repository/search_repository_base.py +++ b/src/basic_memory/repository/search_repository_base.py @@ -998,13 +998,13 @@ async def index_item(self, search_index_row: SearchIndexRow) -> None: await session.execute( text(""" INSERT INTO search_index ( - id, title, content_stems, content_snippet, permalink, file_path, type, metadata, + id, title, content_stems, content_snippet, search_tokens, permalink, file_path, type, metadata, from_id, to_id, relation_type, entity_id, category, created_at, updated_at, project_id ) VALUES ( - :id, :title, :content_stems, :content_snippet, :permalink, :file_path, :type, :metadata, + :id, :title, :content_stems, :content_snippet, :search_tokens, :permalink, :file_path, :type, :metadata, :from_id, :to_id, :relation_type, :entity_id, :category, :created_at, :updated_at, @@ -1045,13 +1045,13 @@ async def bulk_index_items(self, search_index_rows: List[SearchIndexRow]) -> Non await session.execute( text(""" INSERT INTO search_index ( - id, title, content_stems, content_snippet, permalink, file_path, type, metadata, + id, title, content_stems, content_snippet, search_tokens, permalink, file_path, type, metadata, from_id, to_id, relation_type, entity_id, category, created_at, updated_at, project_id ) VALUES ( - :id, :title, :content_stems, :content_snippet, :permalink, :file_path, :type, :metadata, + :id, :title, :content_stems, :content_snippet, :search_tokens, :permalink, :file_path, :type, :metadata, :from_id, :to_id, :relation_type, :entity_id, :category, :created_at, :updated_at, @@ -1068,7 +1068,7 @@ async def get_entity_search_rows(self, entity_id: int) -> list[SearchIndexRow]: async with db.scoped_session(self.session_maker) as session: result = await session.execute( text( - "SELECT project_id, id, title, content_stems, content_snippet, " + "SELECT project_id, id, title, content_stems, content_snippet, search_tokens, " "permalink, file_path, type, metadata, from_id, to_id, relation_type, " "entity_id, category, created_at, updated_at " "FROM search_index " @@ -2432,7 +2432,7 @@ async def _fetch_search_index_rows_by_ids( sql = f""" SELECT project_id, id, title, permalink, file_path, type, metadata, - from_id, to_id, relation_type, entity_id, content_snippet, + from_id, to_id, relation_type, entity_id, content_snippet, search_tokens, category, created_at, updated_at, 0 as score FROM search_index WHERE project_id = :project_id diff --git a/src/basic_memory/services/search_service.py b/src/basic_memory/services/search_service.py index ea5ad89ec..0841651f7 100644 --- a/src/basic_memory/services/search_service.py +++ b/src/basic_memory/services/search_service.py @@ -22,7 +22,7 @@ SearchIndexRow, SearchRepository, ) -from basic_memory.repository.search_query import relaxed_query_words +from basic_memory.repository.search_query import cjk_search_tokens, relaxed_query_words from basic_memory.repository.search_trace import SearchTraceCollector from basic_memory.schemas.base import normalize_note_type from basic_memory.schemas.search import SearchQuery, SearchItemType, SearchRetrievalMode @@ -782,12 +782,14 @@ async def index_entity_file( entity: Entity, ) -> None: # Index entity file with no content + title = _strip_nul(entity.title) await self.repository.index_item( SearchIndexRow( id=entity.id, entity_id=entity.id, type=SearchItemType.ENTITY.value, - title=_strip_nul(entity.title), + title=title, + search_tokens=cjk_search_tokens(title, entity.permalink), permalink=entity.permalink, # Required for Postgres NOT NULL constraint file_path=entity.file_path, metadata={ @@ -862,13 +864,17 @@ async def index_entity_markdown( :MAX_CONTENT_STEMS_SIZE ] # pragma: no cover + entity_title = _strip_nul(entity.title) rows_to_index.append( SearchIndexRow( id=entity.id, type=SearchItemType.ENTITY.value, - title=_strip_nul(entity.title), + title=entity_title, content_stems=entity_content_stems, content_snippet=content_snippet, + search_tokens=cjk_search_tokens( + entity_title, entity.permalink, entity_content_stems + ), permalink=entity.permalink, file_path=entity.file_path, entity_id=entity.id, @@ -896,13 +902,18 @@ async def index_entity_markdown( obs_content_stems = obs_content_stems[ :MAX_CONTENT_STEMS_SIZE ] # pragma: no cover + obs_title = _strip_nul(f"{obs.category}: {obs.content[:100]}...") + obs_content_snippet = _strip_nul(obs.content) rows_to_index.append( SearchIndexRow( id=obs.id, type=SearchItemType.OBSERVATION.value, - title=_strip_nul(f"{obs.category}: {obs.content[:100]}..."), + title=obs_title, content_stems=obs_content_stems, - content_snippet=_strip_nul(obs.content), + content_snippet=obs_content_snippet, + search_tokens=cjk_search_tokens( + obs_title, obs_permalink, obs_content_stems + ), permalink=obs_permalink, file_path=entity.file_path, category=obs.category, @@ -932,6 +943,9 @@ async def index_entity_markdown( title=relation_title, permalink=rel.permalink, content_stems=rel_content_stems, + search_tokens=cjk_search_tokens( + relation_title, rel.permalink, rel_content_stems + ), file_path=entity.file_path, type=SearchItemType.RELATION.value, entity_id=entity.id, diff --git a/tests/repository/test_postgres_search_repository.py b/tests/repository/test_postgres_search_repository.py index 7682994da..b875c0644 100644 --- a/tests/repository/test_postgres_search_repository.py +++ b/tests/repository/test_postgres_search_repository.py @@ -12,6 +12,7 @@ from basic_memory import db from basic_memory.config import BasicMemoryConfig, DatabaseBackend import basic_memory.repository.search_repository_base as search_repository_base_module +import basic_memory.repository.postgres_search_repository as search_repository_module from basic_memory.repository.litellm_provider import LiteLLMEmbeddingProvider from basic_memory.repository.postgres_search_repository import ( PostgresSearchRepository, @@ -477,6 +478,57 @@ async def test_index_item_strips_nul_bytes(session_maker, test_project): assert "\x00" not in (results[0].title or "") +@pytest.mark.asyncio +async def test_postgres_chunk_tokens_are_derived_per_chunk( + session_maker, + test_project, + monkeypatch: pytest.MonkeyPatch, +): + """Chunk token streams must not synthesize bigrams across chunk boundaries.""" + monkeypatch.setattr(search_repository_module, "POSTGRES_FTS_CHUNK_SIZE", 6) + monkeypatch.setattr(search_repository_module, "POSTGRES_FTS_CHUNK_OVERLAP", 2) + + repo = PostgresSearchRepository(session_maker, project_id=test_project.id) + now = datetime.now(timezone.utc) + await repo.index_item( + SearchIndexRow( + project_id=test_project.id, + id=97, + title="Chunked CJK", + content_stems="chunked cjk", + content_snippet="甲乙丙丁\n\n戊己庚辛", + permalink="test/chunked-cjk", + file_path="test/chunked-cjk.md", + type="entity", + metadata={"note_type": "note"}, + created_at=now, + updated_at=now, + ) + ) + + async with db.scoped_session(session_maker) as session: + chunks = ( + ( + await session.execute( + text( + "SELECT chunk_index, chunk_text, chunk_tokens " + "FROM search_index_fts_chunks " + "WHERE project_id = :project_id AND search_index_id = :id " + "ORDER BY chunk_index" + ), + {"project_id": test_project.id, "id": 97}, + ) + ) + .mappings() + .all() + ) + + assert [(chunk["chunk_text"], chunk["chunk_tokens"]) for chunk in chunks] == [ + ("甲乙丙丁\n\n", "甲乙 乙丙 丙丁"), + ("\n\n戊己庚辛", "戊己 己庚 庚辛"), + ] + + def test_strip_nul_from_row(): """_strip_nul_from_row strips NUL bytes from string values, leaves non-strings alone.""" row = { diff --git a/tests/repository/test_search_repository.py b/tests/repository/test_search_repository.py index 8eb049033..5177a8586 100644 --- a/tests/repository/test_search_repository.py +++ b/tests/repository/test_search_repository.py @@ -545,6 +545,117 @@ async def test_to_insert_includes_project_id(search_repository): assert insert_data["project_id"] == search_repository.project_id +@pytest.mark.asyncio +async def test_index_item_persists_search_tokens_and_existing_fields( + search_repository, + search_entity, +): + """Single-row persistence must write the auxiliary token column independently.""" + tokens = "标题 题甲 目录 录乙" + row = SearchIndexRow( + id=search_entity.id, + type=SearchItemType.ENTITY.value, + title="标题甲", + content_stems="标题甲 original stems", + content_snippet="Original display content", + search_tokens=tokens, + permalink=search_entity.permalink, + file_path=search_entity.file_path, + entity_id=search_entity.id, + metadata={"note_type": search_entity.note_type}, + created_at=search_entity.created_at, + updated_at=search_entity.updated_at, + project_id=search_repository.project_id, + ) + + await search_repository.index_item(row) + + async with db.scoped_session(search_repository.session_maker) as session: + persisted = ( + ( + await session.execute( + text( + "SELECT title, content_stems, content_snippet, search_tokens " + "FROM search_index WHERE project_id = :project_id AND id = :id" + ), + {"project_id": search_repository.project_id, "id": search_entity.id}, + ) + ) + .mappings() + .one() + ) + + assert persisted["title"] == row.title + assert persisted["content_stems"] == row.content_stems + assert persisted["content_snippet"] == row.content_snippet + assert persisted["search_tokens"] == tokens + + +@pytest.mark.asyncio +async def test_bulk_index_items_persists_search_tokens(search_repository, search_entity): + """Bulk persistence must bind search tokens for every row in the batch.""" + now = datetime.now(timezone.utc) + rows = [ + SearchIndexRow( + id=search_entity.id, + type=SearchItemType.ENTITY.value, + title="甲标题", + content_stems="甲标题 stems", + content_snippet="first display", + search_tokens="甲标 标题", + permalink=search_entity.permalink, + file_path=search_entity.file_path, + entity_id=search_entity.id, + metadata={"note_type": search_entity.note_type}, + created_at=now, + updated_at=now, + project_id=search_repository.project_id, + ), + SearchIndexRow( + id=search_entity.id + 1, + type=SearchItemType.ENTITY.value, + title="乙标题", + content_stems="乙标题 stems", + content_snippet="second display", + search_tokens="乙标 标题", + permalink="test/search-test-entity-two", + file_path="test/search_test_entity_two.md", + entity_id=search_entity.id + 1, + metadata={"note_type": search_entity.note_type}, + created_at=now, + updated_at=now, + project_id=search_repository.project_id, + ), + ] + + await search_repository.bulk_index_items(rows) + + async with db.scoped_session(search_repository.session_maker) as session: + persisted = ( + ( + await session.execute( + text( + "SELECT id, content_snippet, search_tokens FROM search_index " + "WHERE project_id = :project_id AND id IN (:first_id, :second_id) " + "ORDER BY id" + ), + { + "project_id": search_repository.project_id, + "first_id": search_entity.id, + "second_id": search_entity.id + 1, + }, + ) + ) + .mappings() + .all() + ) + + assert [(row["content_snippet"], row["search_tokens"]) for row in persisted] == [ + ("first display", "甲标 标题"), + ("second display", "乙标 标题"), + ] + + def test_directory_property(): """Test the directory property of SearchIndexRow.""" # Test a file in a nested directory diff --git a/tests/services/test_search_service.py b/tests/services/test_search_service.py index 230bfc733..4fb74cc0a 100644 --- a/tests/services/test_search_service.py +++ b/tests/services/test_search_service.py @@ -442,6 +442,73 @@ async def test_update_index(search_service, full_entity): assert len(results) > 1 +@pytest.mark.asyncio +async def test_index_entity_persists_cjk_search_tokens_without_changing_display_fields( + search_service, + session_maker, + test_project, +): + """Entity indexing stores CJK bigrams separately from the display fields.""" + from basic_memory.repository import EntityRepository + + entity_repository = EntityRepository(project_id=test_project.id) + title = "标题甲" + permalink = "目录/链接乙" + content = "正文丙混合 latinword" + entity = await _create_entity( + session_maker, + entity_repository, + { + "title": title, + "note_type": "note", + "entity_metadata": {}, + "content_type": "text/markdown", + "file_path": "cjk/display-integrity.md", + "permalink": permalink, + "project_id": test_project.id, + "created_at": datetime.now(timezone.utc), + "updated_at": datetime.now(timezone.utc), + }, + ) + + await search_service.index_entity(entity, content=content) + + async with db.scoped_session(session_maker) as session: + row = ( + ( + await session.execute( + text( + "SELECT title, permalink, content_stems, content_snippet, search_tokens " + "FROM search_index WHERE project_id = :project_id AND id = :id" + ), + {"project_id": test_project.id, "id": entity.id}, + ) + ) + .mappings() + .one() + ) + + assert row["title"] == title + assert row["permalink"] == permalink + assert row["content_snippet"] == content + assert title in row["content_stems"] + assert content in row["content_stems"] + assert row["search_tokens"] is not None + assert { + "标题", + "题甲", + "目录", + "链接", + "接乙", + "正文", + "文丙", + "丙混", + "混合", + } <= set(row["search_tokens"].split()) + assert "latinword" not in row["search_tokens"] + assert "甲链" not in row["search_tokens"] + + @pytest.mark.asyncio async def test_boolean_and_search(search_service, test_graph): """Test boolean AND search.""" From 2ba2e54458cffb9ceebbc3885bb51e1dd305094d Mon Sep 17 00:00:00 2001 From: Aryan Pardeshi Date: Sat, 29 Aug 2026 03:48:12 +0530 Subject: [PATCH 5/7] fix(core): complete CJK token write paths Signed-off-by: Aryan Pardeshi --- .../indexing/accepted_note_search.py | 20 ++++--- .../accepted_note_search_repository.py | 10 ++-- .../repository/accepted_note_search_row.py | 1 + .../repository/postgres_search_repository.py | 3 +- tests/indexing/test_accepted_note_search.py | 36 ++++++++++++ .../test_accepted_note_search_repository.py | 51 +++++++++++++++++ .../test_postgres_search_repository_sql.py | 56 +++++++++++++++++++ 7 files changed, 164 insertions(+), 13 deletions(-) create mode 100644 tests/repository/test_postgres_search_repository_sql.py diff --git a/src/basic_memory/indexing/accepted_note_search.py b/src/basic_memory/indexing/accepted_note_search.py index d1e51e29e..c259e4dc8 100644 --- a/src/basic_memory/indexing/accepted_note_search.py +++ b/src/basic_memory/indexing/accepted_note_search.py @@ -9,6 +9,7 @@ from basic_memory.file_utils import ParseError, remove_frontmatter from basic_memory.repository.accepted_note_search_row import AcceptedNoteSearchRow +from basic_memory.repository.search_query import cjk_search_tokens from basic_memory.schemas.base import normalize_note_type MAX_ACCEPTED_SEARCH_CONTENT_STEMS_SIZE = 6000 @@ -112,17 +113,20 @@ def build_accepted_note_search_row( item_type: str = "entity", ) -> AcceptedNoteSearchRow: """Build the hot entity search row for one accepted note snapshot.""" + title_text = strip_search_text(title) + content_stems = accepted_note_content_stems( + title=title, + search_content=search_content, + permalink=permalink, + file_path=file_path, + tags=accepted_note_tags(entity_metadata), + ) return AcceptedNoteSearchRow( id=entity_id, - title=strip_search_text(title), - content_stems=accepted_note_content_stems( - title=title, - search_content=search_content, - permalink=permalink, - file_path=file_path, - tags=accepted_note_tags(entity_metadata), - ), + title=title_text, + content_stems=content_stems, content_snippet=strip_search_text(search_content), + search_tokens=cjk_search_tokens(title_text, permalink, content_stems), permalink=permalink, file_path=Path(file_path).as_posix(), item_type=item_type, diff --git a/src/basic_memory/repository/accepted_note_search_repository.py b/src/basic_memory/repository/accepted_note_search_repository.py index 30d06bdd9..96e1d878a 100644 --- a/src/basic_memory/repository/accepted_note_search_repository.py +++ b/src/basic_memory/repository/accepted_note_search_repository.py @@ -28,13 +28,13 @@ INSERT_ACCEPTED_NOTE_SEARCH_SQL = text( """ INSERT INTO search_index ( - id, title, content_stems, content_snippet, permalink, file_path, type, metadata, + id, title, content_stems, content_snippet, search_tokens, permalink, file_path, type, metadata, from_id, to_id, relation_type, entity_id, category, created_at, updated_at, project_id ) VALUES ( - :id, :title, :content_stems, :content_snippet, :permalink, :file_path, :type, + :id, :title, :content_stems, :content_snippet, :search_tokens, :permalink, :file_path, :type, :metadata, NULL, NULL, NULL, :entity_id, NULL, @@ -47,13 +47,13 @@ UPSERT_ACCEPTED_NOTE_SEARCH_SQL = text( """ INSERT INTO search_index ( - id, title, content_stems, content_snippet, permalink, file_path, type, metadata, + id, title, content_stems, content_snippet, search_tokens, permalink, file_path, type, metadata, from_id, to_id, relation_type, entity_id, category, created_at, updated_at, project_id ) VALUES ( - :id, :title, :content_stems, :content_snippet, :permalink, :file_path, :type, + :id, :title, :content_stems, :content_snippet, :search_tokens, :permalink, :file_path, :type, CAST(:metadata AS jsonb), NULL, NULL, NULL, :entity_id, NULL, @@ -65,6 +65,7 @@ title = EXCLUDED.title, content_stems = EXCLUDED.content_stems, content_snippet = EXCLUDED.content_snippet, + search_tokens = EXCLUDED.search_tokens, file_path = EXCLUDED.file_path, type = EXCLUDED.type, metadata = EXCLUDED.metadata, @@ -95,6 +96,7 @@ def accepted_note_search_insert_params( "title": row.title, "content_stems": row.content_stems, "content_snippet": row.content_snippet, + "search_tokens": row.search_tokens, "permalink": row.permalink, "file_path": row.file_path, "type": row.item_type, diff --git a/src/basic_memory/repository/accepted_note_search_row.py b/src/basic_memory/repository/accepted_note_search_row.py index ff3348646..4cbdd3994 100644 --- a/src/basic_memory/repository/accepted_note_search_row.py +++ b/src/basic_memory/repository/accepted_note_search_row.py @@ -12,6 +12,7 @@ class AcceptedNoteSearchRow: title: str content_stems: str content_snippet: str + search_tokens: str permalink: str | None file_path: str item_type: str diff --git a/src/basic_memory/repository/postgres_search_repository.py b/src/basic_memory/repository/postgres_search_repository.py index e292fd4cd..75959b380 100644 --- a/src/basic_memory/repository/postgres_search_repository.py +++ b/src/basic_memory/repository/postgres_search_repository.py @@ -323,7 +323,8 @@ async def _replace_fts_chunks( chunk.search_index_id, chunk.search_index_type, chunk.chunk_index, - chunk.chunk_text + chunk.chunk_text, + chunk.chunk_tokens FROM jsonb_to_recordset(CAST(:chunks AS JSONB)) AS chunk( search_index_id INTEGER, search_index_type VARCHAR, diff --git a/tests/indexing/test_accepted_note_search.py b/tests/indexing/test_accepted_note_search.py index cfca1b183..286e511e3 100644 --- a/tests/indexing/test_accepted_note_search.py +++ b/tests/indexing/test_accepted_note_search.py @@ -88,6 +88,7 @@ def test_build_accepted_note_search_row_returns_immutable_hot_search_state() -> title="Project Plan", content_stems=row.content_stems, content_snippet="Main body", + search_tokens="", permalink="main/project-plan", file_path="notes/project-plan.md", item_type="entity", @@ -104,6 +105,41 @@ def test_build_accepted_note_search_row_returns_immutable_hot_search_state() -> setattr(row, "title", "Changed") +def test_build_accepted_note_search_row_derives_cjk_tokens_without_changing_display_fields() -> ( + None +): + """Accepted rows keep CJK lexical tokens separate from user-visible text.""" + row = build_accepted_note_search_row( + entity_id=42, + title="标题甲", + note_type="decision", + entity_metadata=None, + permalink="目录/链接乙", + file_path="notes/display.md", + search_content="正文丙混合 latinword", + created_at=datetime(2026, 6, 18, 12, 0, tzinfo=UTC), + updated_at=datetime(2026, 6, 18, 13, 0, tzinfo=UTC), + project_id=7, + ) + + assert { + "标题", + "题甲", + "目录", + "链接", + "接乙", + "正文", + "文丙", + "丙混", + "混合", + } <= set(row.search_tokens.split()) + assert "latinword" not in row.search_tokens + assert "甲目" not in row.search_tokens + assert row.title == "标题甲" + assert row.permalink == "目录/链接乙" + assert row.content_snippet == "正文丙混合 latinword" + + def test_build_accepted_note_search_row_canonicalizes_legacy_note_type() -> None: timestamp = datetime(2026, 6, 18, 12, 0, tzinfo=UTC) diff --git a/tests/repository/test_accepted_note_search_repository.py b/tests/repository/test_accepted_note_search_repository.py index bbdf3bf0d..0f8651ff0 100644 --- a/tests/repository/test_accepted_note_search_repository.py +++ b/tests/repository/test_accepted_note_search_repository.py @@ -9,6 +9,7 @@ from basic_memory.indexing.accepted_note_search import build_accepted_note_search_row from basic_memory.repository.accepted_note_search_repository import ( AcceptedNoteSearchRepository, + accepted_note_search_insert_params, ) @@ -62,11 +63,14 @@ async def test_refresh_entity_replaces_project_scoped_hot_search_row() -> None: assert delete_params == {"entity_id": 42, "project_id": 7} assert "CAST(:metadata AS jsonb)" in insert_sql assert "ON CONFLICT (permalink, project_id)" in insert_sql + assert "search_tokens" in insert_sql + assert "search_tokens = EXCLUDED.search_tokens" in insert_sql assert insert_params == { "id": 42, "title": "Project Plan", "content_stems": row.content_stems, "content_snippet": "Main body", + "search_tokens": "", "permalink": "main/project-plan", "file_path": "notes/project-plan.md", "type": "entity", @@ -102,6 +106,53 @@ async def test_refresh_entity_uses_plain_insert_for_sqlite_virtual_table() -> No assert "ON CONFLICT" not in insert_sql assert "CAST(:metadata AS jsonb)" not in insert_sql assert ":metadata" in insert_sql + assert "search_tokens" in insert_sql + + +@pytest.mark.parametrize("dialect_name", ["sqlite", "postgresql"]) +@pytest.mark.asyncio +async def test_refresh_entity_binds_cjk_tokens_without_cross_field_or_latin_text( + dialect_name: str, +) -> None: + """Both accepted-note SQL paths receive index-only CJK tokens and intact fields.""" + repository = AcceptedNoteSearchRepository(project_id=7) + session = _RecordingSession(dialect_name=dialect_name) + row = build_accepted_note_search_row( + entity_id=42, + title="标题甲", + note_type="decision", + entity_metadata=None, + permalink="目录/链接乙", + file_path="notes/display.md", + search_content="正文丙混合 latinword", + created_at=datetime(2026, 6, 18, 12, 0, tzinfo=UTC), + updated_at=datetime(2026, 6, 18, 13, 0, tzinfo=UTC), + project_id=7, + ) + + params = accepted_note_search_insert_params(row) + await repository.refresh_entity(cast(AsyncSession, session), row) + + insert_sql, insert_params = session.executed[1] + assert insert_params == params + assert { + "标题", + "题甲", + "目录", + "链接", + "接乙", + "正文", + "文丙", + "丙混", + "混合", + } <= set(insert_params["search_tokens"].split()) + assert "latinword" not in insert_params["search_tokens"] + assert "甲目" not in insert_params["search_tokens"] + assert insert_params["title"] == row.title + assert insert_params["permalink"] == row.permalink + assert insert_params["content_stems"] == row.content_stems + assert insert_params["content_snippet"] == row.content_snippet + assert "search_tokens" in insert_sql @pytest.mark.asyncio diff --git a/tests/repository/test_postgres_search_repository_sql.py b/tests/repository/test_postgres_search_repository_sql.py new file mode 100644 index 000000000..3d9e2de72 --- /dev/null +++ b/tests/repository/test_postgres_search_repository_sql.py @@ -0,0 +1,56 @@ +"""Docker-free assertions for PostgreSQL search SQL projections.""" + +import json +import re +from datetime import UTC, datetime +from typing import Any, cast + +import pytest +from sqlalchemy.ext.asyncio import AsyncSession + +from basic_memory.repository.postgres_search_repository import PostgresSearchRepository +from basic_memory.repository.search_index_row import SearchIndexRow + + +class _RecordingSession: + def __init__(self) -> None: + self.executed: list[tuple[str, dict[str, Any]]] = [] + + async def execute(self, statement: Any, params: dict[str, Any]) -> None: + self.executed.append((str(statement), params)) + + +@pytest.mark.asyncio +async def test_chunk_insert_projects_every_json_recordset_column() -> None: + """The chunk INSERT keeps its target and SELECT projections one-to-one.""" + repository = PostgresSearchRepository.__new__(PostgresSearchRepository) + repository.project_id = 7 + session = _RecordingSession() + row = SearchIndexRow( + project_id=7, + id=42, + type="entity", + file_path="notes/cjk.md", + content_snippet="甲乙丙", + created_at=datetime(2026, 6, 18, tzinfo=UTC), + updated_at=datetime(2026, 6, 18, tzinfo=UTC), + ) + + await repository._replace_fts_chunks(cast(AsyncSession, session), [row]) + + insert_sql, params = session.executed[1] + target_match = re.search( + r"INSERT INTO search_index_fts_chunks\s*\((.*?)\)\s*SELECT", + insert_sql, + re.DOTALL, + ) + projection_match = re.search(r"\)\s*SELECT\s+(.*?)\s+FROM", insert_sql, re.DOTALL) + assert target_match is not None + assert projection_match is not None + target_columns = tuple(column.strip() for column in target_match.group(1).split(",")) + projected_values = tuple(value.strip() for value in projection_match.group(1).split(",")) + + assert len(target_columns) == len(projected_values) + assert target_columns[-1] == "chunk_tokens" + assert projected_values[-1] == "chunk.chunk_tokens" + assert json.loads(params["chunks"])[0]["chunk_tokens"] == "甲乙 乙丙" From 43c5c91291fc2f3743959cc42006146d23235464 Mon Sep 17 00:00:00 2001 From: Aryan Pardeshi Date: Sun, 30 Aug 2026 00:24:38 +0530 Subject: [PATCH 6/7] feat(core): search CJK terms across lexical indexes Signed-off-by: Aryan Pardeshi --- .../repository/postgres_search_repository.py | 105 +++++++- .../repository/sqlite_search_repository.py | 38 ++- test-int/mcp/test_search_integration.py | 31 +++ tests/repository/test_search_repository.py | 239 ++++++++++++++++++ 4 files changed, 405 insertions(+), 8 deletions(-) diff --git a/src/basic_memory/repository/postgres_search_repository.py b/src/basic_memory/repository/postgres_search_repository.py index 75959b380..e60aec481 100644 --- a/src/basic_memory/repository/postgres_search_repository.py +++ b/src/basic_memory/repository/postgres_search_repository.py @@ -20,7 +20,13 @@ from basic_memory.repository.rerank_provider import RerankProvider from basic_memory.repository.rerank_provider_factory import create_rerank_provider from basic_memory.repository.search_index_row import SearchIndexRow -from basic_memory.repository.search_query import cjk_search_tokens, relaxed_query_words +from basic_memory.repository.search_query import ( + RELAXATION_CJK_PATTERN, + cjk_bigram_tokens, + cjk_search_tokens, + contains_cjk, + relaxed_query_words, +) from basic_memory.repository.semantic_chunking import VectorChunkRecord from basic_memory.repository.search_repository_base import ( SearchRepositoryBase, @@ -364,6 +370,24 @@ def _prepare_search_term(self, term: str, is_prefix: bool = True) -> str: # For non-Boolean queries, prepare single term return self._prepare_single_term(term, is_prefix) + @staticmethod + def _cjk_tsquery_phrase(term: str) -> str: + """Render one CJK run as an adjacent simple-parser tsquery phrase.""" + return " <-> ".join(f"{token}:*" for token in cjk_bigram_tokens(term)) + + @staticmethod + def _cjk_tsquery_text(search_text: str, *, relaxed: bool = False) -> str: + """Render all CJK runs in a query for the auxiliary simple tsvector.""" + phrases = [ + PostgresSearchRepository._cjk_tsquery_phrase(match.group(0)) + for match in RELAXATION_CJK_PATTERN.finditer(search_text) + ] + if relaxed: + return " | ".join(phrases) + if re.search(r"\bOR\b", search_text, flags=re.IGNORECASE): + return " | ".join(phrases) + return " & ".join(phrases) + @staticmethod def _relaxed_tsquery_term(word: str) -> str: """Render one relaxed word as a tsquery-safe prefix expression. @@ -829,6 +853,7 @@ async def _build_fts_query_parts( order_by_clause = "" from_clause = "search_index" document_vector_sql: str | None = None + cjk_text: str | None = None # Handle text search for title and content using tsvector if search_text: @@ -839,6 +864,9 @@ async def _build_fts_query_parts( # Prepare search term for tsquery processed_text = self._prepare_search_term(search_text.strip()) params["text"] = processed_text + if contains_cjk(search_text): + cjk_text = self._cjk_tsquery_text(search_text.strip()) + params["cjk_text"] = cjk_text probe_texts = [processed_text] if allow_relaxed: relaxed_text = self._relaxed_tsquery_text(search_text) @@ -852,13 +880,37 @@ async def _build_fts_query_parts( if candidate_operands: params["text_candidate"] = " | ".join(candidate_operands) + cjk_candidate_arms = "" + if cjk_text is not None: + cjk_candidate_arms = """ + UNION + SELECT + candidate_cjk_parent.project_id, + candidate_cjk_parent.id, + candidate_cjk_parent.type + FROM search_index AS candidate_cjk_parent + WHERE candidate_cjk_parent.project_id = :project_id + AND candidate_cjk_parent.search_tokens_index_col + @@ to_tsquery('simple', :cjk_text) + UNION + SELECT + candidate_cjk_chunk.project_id, + candidate_cjk_chunk.search_index_id AS id, + candidate_cjk_chunk.search_index_type AS type + FROM search_index_fts_chunks AS candidate_cjk_chunk + WHERE candidate_cjk_chunk.project_id = :project_id + AND candidate_cjk_chunk.chunk_tokens_index_col + @@ to_tsquery('simple', :cjk_text) + """ + # Trigger: PostgreSQL can extract a required-positive query tree. # Why: OR-ing its operands is a safe indexed superset even when # terms live in different chunks. Pure/optional negation returns # ``T`` and must retain all project rows for correct semantics. # Outcome: ordinary and required-positive NOT queries use both # GIN indexes; only genuinely unindexable negation scans the project. - from_clause = """ + from_clause = ( + """ search_index JOIN ( SELECT candidate_parent.project_id, @@ -886,14 +938,30 @@ async def _build_fts_query_parts( candidate_all.type FROM search_index AS candidate_all WHERE candidate_all.project_id = :project_id - AND querytree(to_tsquery('english', :text)) = 'T' + AND querytree(to_tsquery('english', :text)) = 'T'""" + + cjk_candidate_arms + + """ ) AS fts_candidate ON fts_candidate.project_id = search_index.project_id AND fts_candidate.id = search_index.id AND fts_candidate.type = search_index.type """ + ) document_vector_sql = self._document_fts_vector_sql(probe_texts, params) - conditions.append(f"{document_vector_sql} @@ to_tsquery('english', :text)") + lexical_condition = f"{document_vector_sql} @@ to_tsquery('english', :text)" + if cjk_text is not None: + lexical_condition = ( + f"({lexical_condition} OR " + "search_index.search_tokens_index_col " + "@@ to_tsquery('simple', :cjk_text) OR EXISTS (" + "SELECT 1 FROM search_index_fts_chunks AS cjk_chunk " + "WHERE cjk_chunk.project_id = search_index.project_id " + "AND cjk_chunk.search_index_id = search_index.id " + "AND cjk_chunk.search_index_type = search_index.type " + "AND cjk_chunk.chunk_tokens_index_col " + "@@ to_tsquery('simple', :cjk_text)))" + ) + conditions.append(lexical_condition) # Handle title search if title: @@ -1053,10 +1121,25 @@ async def _build_fts_query_parts( # Note: If no text search, score will be NULL, so we use COALESCE to default to 0 if search_text and search_text.strip() and search_text.strip() != "*": assert document_vector_sql is not None + cjk_score_terms = "" + if cjk_text is not None: + cjk_score_terms = ( + "ts_rank(search_index.search_tokens_index_col, " + "to_tsquery('simple', :cjk_text)), " + "COALESCE((SELECT MAX(ts_rank(" + "fts_chunk.chunk_tokens_index_col, to_tsquery('simple', :cjk_text))) " + "FROM search_index_fts_chunks AS fts_chunk " + "WHERE fts_chunk.project_id = search_index.project_id " + "AND fts_chunk.search_index_id = search_index.id " + "AND fts_chunk.search_index_type = search_index.type " + "AND fts_chunk.chunk_tokens_index_col " + "@@ to_tsquery('simple', :cjk_text)), 0), " + ) score_expr = ( "GREATEST(" f"ts_rank({document_vector_sql}, to_tsquery('english', :text)), " "ts_rank(search_index.textsearchable_index_col, to_tsquery('english', :text)), " + f"{cjk_score_terms}" "COALESCE((SELECT MAX(ts_rank(" "fts_chunk.textsearchable_index_col, to_tsquery('english', :text))) " "FROM search_index_fts_chunks AS fts_chunk " @@ -1247,9 +1330,14 @@ async def run_search(active_session: AsyncSession): limit=limit, offset=offset, ): + retry_params = {**params, "text": relaxed} + if "cjk_text" in retry_params: + retry_params["cjk_text"] = self._cjk_tsquery_text( + search_text or "", relaxed=True + ) rows = await execute_rows( active_session, - {**params, "text": relaxed}, + retry_params, ) return rows, relaxed_fallback_used @@ -1378,9 +1466,14 @@ async def execute_count(active_session: AsyncSession, query_params: dict[str, An reason="syntax_error" if strict_syntax_error else "empty_result", token_count=len(relaxed_query_words(search_text) or ()), ): + retry_params = {**params, "text": relaxed} + if "cjk_text" in retry_params: + retry_params["cjk_text"] = self._cjk_tsquery_text( + search_text or "", relaxed=True + ) total = await execute_count( session, - {**params, "text": relaxed}, + retry_params, ) return total except Exception as e: diff --git a/src/basic_memory/repository/sqlite_search_repository.py b/src/basic_memory/repository/sqlite_search_repository.py index 9dced81f9..772a8533c 100644 --- a/src/basic_memory/repository/sqlite_search_repository.py +++ b/src/basic_memory/repository/sqlite_search_repository.py @@ -27,7 +27,12 @@ from basic_memory.repository.rerank_provider import RerankProvider from basic_memory.repository.rerank_provider_factory import create_rerank_provider from basic_memory.repository.search_index_row import SearchIndexRow -from basic_memory.repository.search_query import relaxed_query_words +from basic_memory.repository.search_query import ( + RELAXATION_CJK_PATTERN, + cjk_bigram_tokens, + contains_cjk, + relaxed_query_words, +) from basic_memory.repository.search_repository_base import SearchRepositoryBase from basic_memory.repository.search_trace import ( SearchTraceCollector, @@ -397,6 +402,25 @@ def _prepare_search_term(self, term: str, is_prefix: bool = True) -> str: # For non-Boolean queries, use the single term preparation logic return self._prepare_single_term(term, is_prefix) + @staticmethod + def _cjk_fts_phrase(term: str) -> str: + """Render one CJK run as an adjacent FTS5 phrase of overlapping bigrams.""" + tokens = cjk_bigram_tokens(term) + return f'"{" ".join(tokens)}"*' + + @staticmethod + def _cjk_fts_text(search_text: str, *, relaxed: bool = False) -> str: + """Render all CJK runs in a query for the auxiliary FTS5 column.""" + phrases = [ + SQLiteSearchRepository._cjk_fts_phrase(match.group(0)) + for match in RELAXATION_CJK_PATTERN.finditer(search_text) + ] + if relaxed: + return " OR ".join(phrases) + if re.search(r"\bOR\b", search_text, flags=re.IGNORECASE): + return " OR ".join(phrases) + return " AND ".join(phrases) + @staticmethod def _relaxed_fts_term(word: str) -> str: """Render one relaxed word as an FTS5-safe prefix expression. @@ -798,10 +822,16 @@ async def _build_fts_query_parts( params["text"] = processed_text # content_stems is capped for Postgres index-row compatibility, while # SQLite stores the complete note body in its FTS5 content_snippet column. - match_conditions.append( + lexical_condition = ( "(search_index.title MATCH :text OR search_index.content_stems MATCH :text " "OR search_index.content_snippet MATCH :text)" ) + if contains_cjk(search_text): + params["cjk_text"] = self._cjk_fts_text(search_text.strip()) + lexical_condition = ( + f"({lexical_condition} OR search_index.search_tokens MATCH :cjk_text)" + ) + match_conditions.append(lexical_condition) # Handle title match search if title: @@ -1093,6 +1123,8 @@ async def run_search(active_session: AsyncSession): if relaxed and params.get("text"): relaxed_fallback_used = True params["text"] = relaxed + if params.get("cjk_text"): + params["cjk_text"] = self._cjk_fts_text(search_text or "", relaxed=True) logger.debug( "Strict SQLite FTS returned 0 results; retrying relaxed FTS query " f"strict='{search_text}' relaxed='{relaxed}'" @@ -1209,6 +1241,8 @@ async def count( ) if relaxed and params.get("text"): params["text"] = relaxed + if params.get("cjk_text"): + params["cjk_text"] = self._cjk_fts_text(search_text or "", relaxed=True) with logfire.span( "search.count.relaxed_fts_retry", backend="sqlite", diff --git a/test-int/mcp/test_search_integration.py b/test-int/mcp/test_search_integration.py index f7bb9e78d..4bf97b91c 100644 --- a/test-int/mcp/test_search_integration.py +++ b/test-int/mcp/test_search_integration.py @@ -549,3 +549,34 @@ async def test_tags_param_vs_tag_query_comma_consistency(mcp_server, app, test_p "tags='alpha,beta' param must behave like the tag: shorthand " f"(both split commas). query_hit={query_hit} param_hit={param_hit}" ) + + +@pytest.mark.asyncio +async def test_search_cjk_mid_run_term_through_public_surface(mcp_server, app, test_project): + """A CJK substring in a real note is retrievable through MCP search.""" + async with Client(mcp_server) as client: + await client.call_tool( + "write_note", + { + "project": test_project.name, + "title": "CJK lexical search", + "directory": "international", + "content": ( + "# CJK lexical search\n\nThe note contains the longer phrase 前置适者生存后缀." + ), + }, + ) + + search_result = await client.call_tool( + "search_notes", + { + "project": test_project.name, + "query": "者生", + "search_type": "text", + }, + ) + + result_content = search_result.content[0] + assert result_content.type == "text" + result_text = result_content.text + assert "CJK lexical search" in result_text diff --git a/tests/repository/test_search_repository.py b/tests/repository/test_search_repository.py index 5177a8586..bd5c64eff 100644 --- a/tests/repository/test_search_repository.py +++ b/tests/repository/test_search_repository.py @@ -11,6 +11,8 @@ from basic_memory.models.project import Project from basic_memory.repository.search_repository import SearchIndexRow from basic_memory.repository.postgres_search_repository import PostgresSearchRepository +from basic_memory.repository.sqlite_search_repository import SQLiteSearchRepository +from basic_memory.repository.search_query import cjk_search_tokens from basic_memory.schemas.search import SearchItemType @@ -699,6 +701,64 @@ class TestSearchTermPreparation: Tests with `[asyncio-sqlite]` or `[asyncio-postgres]` test backend-agnostic functionality. """ + @pytest.mark.parametrize( + ("term", "expected"), + [ + ("适", '"适"*'), + ("适者", '"适者"*'), + ("适者生", '"适者 者生"*'), + ("适者生存", '"适者 者生 生存"*'), + ], + ) + def test_sqlite_cjk_phrase_uses_overlapping_adjacent_tokens( + self, term: str, expected: str + ) -> None: + """SQLite renders every CJK substring as one adjacent FTS5 phrase.""" + assert SQLiteSearchRepository._cjk_fts_phrase(term) == expected + + @pytest.mark.parametrize( + ("term", "expected"), + [ + ("适", "适:*"), + ("适者", "适者:*"), + ("适者生", "适者:* <-> 者生:*"), + ("适者生存", "适者:* <-> 者生:* <-> 生存:*"), + ], + ) + def test_postgres_cjk_phrase_uses_tsquery_adjacency(self, term: str, expected: str) -> None: + """PostgreSQL joins overlapping CJK bigrams with positional adjacency.""" + assert PostgresSearchRepository._cjk_tsquery_phrase(term) == expected + + @pytest.mark.parametrize( + ("search_text", "relaxed", "expected"), + [ + ("季度 报告", False, '"季度"* AND "报告"*'), + ("季度 OR 报告", False, '"季度"* OR "报告"*'), + ("季度 报告", True, '"季度"* OR "报告"*'), + ], + ) + def test_sqlite_cjk_text_joins_multiple_runs_by_boolean_intent( + self, search_text: str, relaxed: bool, expected: str + ) -> None: + """Separate CJK runs in one query AND-join by default; boolean OR or a + relaxed retry switches every run's phrase to OR instead.""" + assert SQLiteSearchRepository._cjk_fts_text(search_text, relaxed=relaxed) == expected + + @pytest.mark.parametrize( + ("search_text", "relaxed", "expected"), + [ + ("季度 报告", False, "季度:* & 报告:*"), + ("季度 OR 报告", False, "季度:* | 报告:*"), + ("季度 报告", True, "季度:* | 报告:*"), + ], + ) + def test_postgres_cjk_text_joins_multiple_runs_by_boolean_intent( + self, search_text: str, relaxed: bool, expected: str + ) -> None: + """Separate CJK runs in one query AND-join by default; boolean OR or a + relaxed retry switches every run's phrase to OR instead.""" + assert PostgresSearchRepository._cjk_tsquery_text(search_text, relaxed=relaxed) == expected + def test_simple_terms_get_prefix_wildcard(self, search_repository): """Simple alphanumeric terms should get prefix matching.""" from basic_memory.repository.postgres_search_repository import PostgresSearchRepository @@ -1052,6 +1112,185 @@ def test_prepare_single_term_empty_input(self, search_repository): assert result3 == "\t\n" # Should return original +@pytest.mark.asyncio +@pytest.mark.parametrize("query", ["适者", "者生", "者生存", "适者生存"]) +async def test_cjk_search_finds_terms_inside_indexed_runs_and_counts( + search_repository, + search_entity, + query: str, +): + """CJK queries find two-to-four-character terms after another CJK token.""" + content = "前置适者生存后缀" + row = SearchIndexRow( + id=search_entity.id, + type=SearchItemType.ENTITY.value, + title="CJK mid-run search", + content_stems=content, + content_snippet=content, + search_tokens=cjk_search_tokens(content), + permalink=search_entity.permalink, + file_path=search_entity.file_path, + entity_id=search_entity.id, + metadata={"note_type": search_entity.note_type}, + created_at=search_entity.created_at, + updated_at=search_entity.updated_at, + project_id=search_repository.project_id, + ) + await search_repository.index_item(row) + + results = await search_repository.search(search_text=query) + + assert [result.id for result in results] == [search_entity.id] + assert await search_repository.count(search_text=query) == 1 + + +@pytest.mark.asyncio +async def test_cjk_search_supports_single_character_title_and_body_terms( + search_repository, + search_entity, +): + """A one-character CJK run remains searchable in both title and body indexes.""" + row = SearchIndexRow( + id=search_entity.id, + type=SearchItemType.ENTITY.value, + title="标题-适", + content_stems="body marker", + content_snippet="body marker", + search_tokens=cjk_search_tokens("标题-适", search_entity.permalink), + permalink=search_entity.permalink, + file_path=search_entity.file_path, + entity_id=search_entity.id, + metadata={"note_type": search_entity.note_type}, + created_at=search_entity.created_at, + updated_at=search_entity.updated_at, + project_id=search_repository.project_id, + ) + await search_repository.index_item(row) + + results = await search_repository.search(search_text="适") + + assert [result.id for result in results] == [search_entity.id] + + +@pytest.mark.asyncio +async def test_cjk_search_requires_adjacent_bigrams_and_keeps_mixed_ascii_operand( + search_repository, + search_entity, +): + """Auxiliary CJK phrases preserve adjacency and do not drop an ASCII AND term.""" + now = datetime.now(timezone.utc) + rows = [ + SearchIndexRow( + id=search_entity.id, + type=SearchItemType.ENTITY.value, + title="CJK and backend", + content_stems="prefix 适者生存 backendmarker", + content_snippet="prefix 适者生存 backendmarker", + search_tokens="适者 者生 生存 backendmarker", + permalink=search_entity.permalink, + file_path=search_entity.file_path, + entity_id=search_entity.id, + metadata={"note_type": search_entity.note_type}, + created_at=now, + updated_at=now, + project_id=search_repository.project_id, + ), + SearchIndexRow( + id=search_entity.id + 1, + type=SearchItemType.ENTITY.value, + title="CJK only", + content_stems="prefix nonadjacent", + content_snippet="prefix nonadjacent", + search_tokens="适者 gap 者生 生存", + permalink="test/cjk-only", + file_path="test/cjk-only.md", + entity_id=search_entity.id + 1, + metadata={"note_type": search_entity.note_type}, + created_at=now, + updated_at=now, + project_id=search_repository.project_id, + ), + SearchIndexRow( + id=search_entity.id + 2, + type=SearchItemType.ENTITY.value, + title="Backend only", + content_stems="backendmarker", + content_snippet="backendmarker", + search_tokens="", + permalink="test/backend-only", + file_path="test/backend-only.md", + entity_id=search_entity.id + 2, + metadata={"note_type": search_entity.note_type}, + created_at=now, + updated_at=now, + project_id=search_repository.project_id, + ), + ] + for row in rows: + await search_repository.index_item(row) + + mixed = await search_repository.search(search_text="适者生存 backendmarker") + boolean = await search_repository.search(search_text="适者生存 AND backendmarker") + + assert search_entity.id in {result.id for result in mixed} + assert search_entity.id in {result.id for result in boolean} + assert search_entity.id + 2 not in {result.id for result in mixed} + assert search_entity.id + 2 not in {result.id for result in boolean} + + separated = SearchIndexRow( + id=search_entity.id + 3, + type=SearchItemType.ENTITY.value, + title="Separated CJK tokens", + content_stems="separated tokens", + content_snippet="separated tokens", + search_tokens="适者 gap 者生 生存", + permalink="test/separated-cjk", + file_path="test/separated-cjk.md", + entity_id=search_entity.id + 3, + metadata={"note_type": search_entity.note_type}, + created_at=now, + updated_at=now, + project_id=search_repository.project_id, + ) + await search_repository.index_item(separated) + + cjk_results = await search_repository.search(search_text="适者生存") + + assert {result.id for result in cjk_results} == {search_entity.id} + + +@pytest.mark.asyncio +async def test_cjk_search_relaxed_retry_uses_auxiliary_tokens( + search_repository, + search_entity, +): + """Relaxed search retains CJK phrase candidates when strict lexical terms miss.""" + content = "前置适者生存后缀" + row = SearchIndexRow( + id=search_entity.id, + type=SearchItemType.ENTITY.value, + title="CJK relaxed search", + content_stems=content, + content_snippet=content, + search_tokens=cjk_search_tokens(content), + permalink=search_entity.permalink, + file_path=search_entity.file_path, + entity_id=search_entity.id, + metadata={"note_type": search_entity.note_type}, + created_at=search_entity.created_at, + updated_at=search_entity.updated_at, + project_id=search_repository.project_id, + ) + await search_repository.index_item(row) + + results = await search_repository.search( + search_text="不存在 适者生存 另一个缺失词", + allow_relaxed=True, + ) + + assert [result.id for result in results] == [search_entity.id] + + async def _index_entity_with_metadata(search_repository, session_maker, title, entity_metadata): slug = "-".join(title.lower().split()) file_path = f"test/{slug}.md" From 9152cf35a19fd09a3bb54e53916f51c2e0a91afe Mon Sep 17 00:00:00 2001 From: Aryan Pardeshi Date: Sun, 30 Aug 2026 00:45:43 +0530 Subject: [PATCH 7/7] docs(core): document CJK search reindex Signed-off-by: Aryan Pardeshi --- CHANGELOG.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c2bd3ddd3..e95040d3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,31 @@ ## Unreleased +### Features + +- **#1294**: Chinese, Japanese, and Korean terms now match anywhere in an + indexed note, not only at the start of a CJK run. Search indexing derives an + index-only stream of overlapping CJK bigrams (`search_tokens` as an extra + SQLite FTS5 column; `search_tokens`/`chunk_tokens` text columns with + `simple`-configuration generated `tsvector`s and GIN indexes on PostgreSQL), + and CJK queries are rendered as exact adjacency phrases against those + columns. Bigrams never cross a field or chunk boundary, so scattered + characters do not satisfy a contiguous term. Display text is unchanged: + titles, permalinks, `content_stems`, snippets, and chunk text keep their + original bytes, and pure non-CJK queries take the same SQL and ranking paths + as before. + + The migration adds schema only — it does not backfill derived tokens. + Existing installations must repopulate the search index once after + upgrading: + + ```text + basic-memory reindex --full --search + ``` + + New installs and any note written after the upgrade index CJK tokens + automatically. + ## v0.23.2 (2026-08-25) Patch release fixing case-duplicate folder creation.