diff --git a/docs/semantic-search.md b/docs/semantic-search.md index 7984f6122..1aaa4af6f 100644 --- a/docs/semantic-search.md +++ b/docs/semantic-search.md @@ -203,17 +203,21 @@ use the same model contract. ### Chinese, Japanese, Korean and other unsegmented scripts -Keyword (full-text) search does not segment CJK text. Both backends tokenize a run of CJK -characters as a single token — SQLite's FTS5 `unicode61` tokenizer and Postgres's default text -search parser alike. Query terms are matched as prefixes, so `生存` finds a note containing -`生存竞争` (the run starts with the query) but not one containing only `适者生存`, where the -query sits in the middle of a run. - -Until a segmenting tokenizer option ships (tracked in -[#1294](https://github.com/basicmachines-co/basic-memory/issues/1294)), use semantic or hybrid -search with a multilingual embedding model for these languages — the Jina Chinese-English model -or multilingual E5 configured above both work — and expect keyword search to match whole runs -and run prefixes only. +Full-text search automatically adds an ordered script n-gram channel for writing systems that do +not reliably separate words with spaces. It covers Han, Hiragana, Katakana, Hangul, Bopomofo, +Thai, Lao, Tibetan, Myanmar, and Khmer text on both SQLite and Postgres. For example, `生存` +matches the middle of `适者生存`, while a query with the characters in a different order does +not. Mixed queries such as `OpenAI 适者生存` require both the word and script terms. + +There is no language, tokenizer, or embedding setting to enable. New and edited notes are indexed +automatically. After upgrading an existing local project, run `bm reindex --full --search` once +to force every existing note through search indexing and populate its script terms. Hosted Cloud +projects receive the same Postgres analysis through the managed full fleet reindex; Cloud users +do not run a local command. + +This is lexical matching, independent of the configured embedding model. Vector and hybrid search +quality in these languages still depends on choosing a multilingual embedding model such as the +Jina Chinese-English model or multilingual E5 configured above. ### OpenAI diff --git a/src/basic_memory/alembic/versions/d2e3f4a5b6c7_add_script_ngrams_to_full_text_search.py b/src/basic_memory/alembic/versions/d2e3f4a5b6c7_add_script_ngrams_to_full_text_search.py new file mode 100644 index 000000000..913297889 --- /dev/null +++ b/src/basic_memory/alembic/versions/d2e3f4a5b6c7_add_script_ngrams_to_full_text_search.py @@ -0,0 +1,123 @@ +"""Add portable script n-grams to full-text search. + +Revision ID: d2e3f4a5b6c7 +Revises: bcdbd5a942ca +Create Date: 2026-08-29 00:00:00.000000 + +""" + +from typing import Sequence, Union + +from alembic import op +from sqlalchemy import text + + +revision: str = "d2e3f4a5b6c7" +down_revision: Union[str, None] = "bcdbd5a942ca" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +SQLITE_COLUMNS = """ + id, title, content_stems, content_snippet, {script_column} permalink, + file_path, type, project_id, from_id, to_id, relation_type, entity_id, + category, metadata, created_at, updated_at +""" + + +def rebuild_sqlite_search_index(*, include_script_ngrams: bool) -> None: + """Copy the FTS5 table while changing its indexed-column contract.""" + bind = op.get_bind() + search_index_sql: str | None = bind.execute( + text("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'search_index'") + ).scalar_one_or_none() + + # The runtime owns creation of the derived FTS table, so migrations only transform an + # existing FTS5 index. This also leaves unrelated physical tables with the same name alone. + if search_index_sql is None or "using fts5" not in search_index_sql.casefold(): + return + + script_definition = "script_ngrams," if include_script_ngrams else "" + op.execute(f""" + CREATE VIRTUAL TABLE search_index_rebuilt USING fts5( + id UNINDEXED, + title, + content_stems, + content_snippet, + {script_definition} + permalink, + file_path UNINDEXED, + type UNINDEXED, + project_id UNINDEXED, + from_id UNINDEXED, + to_id UNINDEXED, + relation_type UNINDEXED, + entity_id UNINDEXED, + category UNINDEXED, + metadata UNINDEXED, + created_at UNINDEXED, + updated_at UNINDEXED, + tokenize='unicode61 tokenchars 0x2F', + prefix='1,2,3,4' + ) + """) + + source_columns = SQLITE_COLUMNS.format(script_column="") + target_columns = SQLITE_COLUMNS.format( + script_column="script_ngrams," if include_script_ngrams else "" + ) + selected_columns = SQLITE_COLUMNS.format(script_column="'' AS script_ngrams,") + if not include_script_ngrams: + selected_columns = source_columns + op.execute( + f"INSERT INTO search_index_rebuilt ({target_columns}) " + f"SELECT {selected_columns} FROM search_index" + ) + op.execute("DROP TABLE search_index") + op.execute("ALTER TABLE search_index_rebuilt RENAME TO search_index") + + +def upgrade() -> None: + """Add the derived script channel; a reindex populates existing rows.""" + if op.get_bind().dialect.name == "sqlite": + rebuild_sqlite_search_index(include_script_ngrams=True) + return + + op.execute("ALTER TABLE search_index ADD COLUMN script_ngrams TEXT NOT NULL DEFAULT ''") + op.execute(""" + ALTER TABLE search_index + ADD COLUMN script_ngrams_index_col tsvector GENERATED ALWAYS AS ( + to_tsvector('simple', script_ngrams) + ) STORED + """) + op.execute(""" + CREATE INDEX idx_search_index_script_ngrams_fts + ON search_index USING gin(script_ngrams_index_col) + """) + op.execute( + "ALTER TABLE search_index_fts_chunks ADD COLUMN script_ngrams TEXT NOT NULL DEFAULT ''" + ) + op.execute(""" + ALTER TABLE search_index_fts_chunks + ADD COLUMN script_ngrams_index_col tsvector GENERATED ALWAYS AS ( + to_tsvector('simple', script_ngrams) + ) STORED + """) + op.execute(""" + CREATE INDEX idx_search_index_fts_chunks_script_ngrams_fts + ON search_index_fts_chunks USING gin(script_ngrams_index_col) + """) + + +def downgrade() -> None: + """Remove the script channel while preserving the existing word index.""" + if op.get_bind().dialect.name == "sqlite": + rebuild_sqlite_search_index(include_script_ngrams=False) + return + + op.execute("DROP INDEX IF EXISTS idx_search_index_fts_chunks_script_ngrams_fts") + op.execute("ALTER TABLE search_index_fts_chunks DROP COLUMN IF EXISTS script_ngrams_index_col") + op.execute("ALTER TABLE search_index_fts_chunks DROP COLUMN IF EXISTS script_ngrams") + op.execute("DROP INDEX IF EXISTS idx_search_index_script_ngrams_fts") + op.execute("ALTER TABLE search_index DROP COLUMN IF EXISTS script_ngrams_index_col") + op.execute("ALTER TABLE search_index DROP COLUMN IF EXISTS script_ngrams") diff --git a/src/basic_memory/models/search.py b/src/basic_memory/models/search.py index c9616d850..5501cd83a 100644 --- a/src/basic_memory/models/search.py +++ b/src/basic_memory/models/search.py @@ -21,6 +21,7 @@ title TEXT, content_stems TEXT, content_snippet TEXT, + script_ngrams TEXT NOT NULL DEFAULT '', permalink VARCHAR, file_path VARCHAR, type VARCHAR, @@ -39,6 +40,9 @@ coalesce(content_stems, '') ) ) STORED, + script_ngrams_index_col tsvector GENERATED ALWAYS AS ( + to_tsvector('simple', script_ngrams) + ) STORED, PRIMARY KEY (id, type, project_id), FOREIGN KEY (project_id) REFERENCES project(id) ON DELETE CASCADE ) @@ -48,6 +52,11 @@ CREATE INDEX IF NOT EXISTS idx_search_index_fts ON search_index USING gin(textsearchable_index_col) """) +CREATE_POSTGRES_SEARCH_INDEX_SCRIPT_NGRAMS_FTS = DDL(""" +CREATE INDEX IF NOT EXISTS idx_search_index_script_ngrams_fts +ON search_index USING gin(script_ngrams_index_col) +""") + # Full note bodies are stored in bounded child rows so one unusually large note # cannot exceed PostgreSQL's per-tsvector size limit. CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_TABLE = DDL(""" @@ -57,9 +66,13 @@ search_index_type VARCHAR NOT NULL, chunk_index INTEGER NOT NULL, chunk_text TEXT NOT NULL, + script_ngrams TEXT NOT NULL DEFAULT '', textsearchable_index_col tsvector GENERATED ALWAYS AS ( to_tsvector('english', chunk_text) ) STORED, + script_ngrams_index_col tsvector GENERATED ALWAYS AS ( + to_tsvector('simple', script_ngrams) + ) STORED, PRIMARY KEY (project_id, search_index_id, search_index_type, chunk_index), FOREIGN KEY (search_index_id, search_index_type, project_id) REFERENCES search_index(id, type, project_id) @@ -72,6 +85,11 @@ ON search_index_fts_chunks USING gin(textsearchable_index_col) """) +CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_SCRIPT_NGRAMS_INDEX = DDL(""" +CREATE INDEX IF NOT EXISTS idx_search_index_fts_chunks_script_ngrams_fts +ON search_index_fts_chunks USING gin(script_ngrams_index_col) +""") + CREATE_POSTGRES_SEARCH_INDEX_METADATA = DDL(""" CREATE INDEX IF NOT EXISTS idx_search_index_metadata_gin ON search_index USING gin(metadata jsonb_path_ops) """) @@ -94,6 +112,7 @@ title, -- Title for searching content_stems, -- Main searchable content split into stems content_snippet, -- File content snippet for display + script_ngrams, -- Portable bigrams for scripts without word boundaries permalink, -- Stable identifier (now indexed for path search) file_path UNINDEXED, -- Physical location type UNINDEXED, -- entity/relation/observation diff --git a/src/basic_memory/repository/accepted_note_search_repository.py b/src/basic_memory/repository/accepted_note_search_repository.py index 30d06bdd9..f566f6e63 100644 --- a/src/basic_memory/repository/accepted_note_search_repository.py +++ b/src/basic_memory/repository/accepted_note_search_repository.py @@ -13,6 +13,8 @@ ProjectIndexExternalVectorCleaner, delete_project_index_vector_rows, ) +from basic_memory.repository.postgres_fts_chunks import split_postgres_fts_chunks +from basic_memory.repository.script_ngrams import build_script_ngrams type SearchIndexSqlValue = str | int | datetime | None type SearchIndexSqlParams = dict[str, SearchIndexSqlValue] @@ -28,14 +30,15 @@ INSERT_ACCEPTED_NOTE_SEARCH_SQL = text( """ INSERT INTO search_index ( - id, title, content_stems, content_snippet, permalink, file_path, type, metadata, + id, title, content_stems, content_snippet, script_ngrams, + permalink, file_path, type, metadata, from_id, to_id, relation_type, entity_id, category, created_at, updated_at, project_id ) VALUES ( - :id, :title, :content_stems, :content_snippet, :permalink, :file_path, :type, - :metadata, + :id, :title, :content_stems, :content_snippet, :script_ngrams, + :permalink, :file_path, :type, :metadata, NULL, NULL, NULL, :entity_id, NULL, :created_at, :updated_at, @@ -47,14 +50,15 @@ UPSERT_ACCEPTED_NOTE_SEARCH_SQL = text( """ INSERT INTO search_index ( - id, title, content_stems, content_snippet, permalink, file_path, type, metadata, + id, title, content_stems, content_snippet, script_ngrams, + permalink, file_path, type, metadata, from_id, to_id, relation_type, entity_id, category, created_at, updated_at, project_id ) VALUES ( - :id, :title, :content_stems, :content_snippet, :permalink, :file_path, :type, - CAST(:metadata AS jsonb), + :id, :title, :content_stems, :content_snippet, :script_ngrams, + :permalink, :file_path, :type, CAST(:metadata AS jsonb), NULL, NULL, NULL, :entity_id, NULL, :created_at, :updated_at, @@ -65,6 +69,7 @@ title = EXCLUDED.title, content_stems = EXCLUDED.content_stems, content_snippet = EXCLUDED.content_snippet, + script_ngrams = EXCLUDED.script_ngrams, file_path = EXCLUDED.file_path, type = EXCLUDED.type, metadata = EXCLUDED.metadata, @@ -78,6 +83,42 @@ """ ) +INSERT_ACCEPTED_NOTE_FTS_CHUNKS_SQL = text( + """ + INSERT INTO search_index_fts_chunks ( + project_id, + search_index_id, + search_index_type, + chunk_index, + chunk_text, + script_ngrams + ) + SELECT + :project_id, + chunk.search_index_id, + chunk.search_index_type, + chunk.chunk_index, + chunk.chunk_text, + chunk.script_ngrams + FROM jsonb_to_recordset(CAST(:chunks AS JSONB)) AS chunk( + search_index_id INTEGER, + search_index_type VARCHAR, + chunk_index INTEGER, + chunk_text TEXT, + script_ngrams TEXT + ) + """ +) + +DELETE_ACCEPTED_NOTE_FTS_CHUNKS_SQL = text( + """ + DELETE FROM search_index_fts_chunks + WHERE project_id = :project_id + AND search_index_id = :search_index_id + AND search_index_type = :search_index_type + """ +) + def accepted_note_search_insert_statement(session: AsyncSession): """Return the insert statement supported by the active search table backend.""" @@ -88,13 +129,19 @@ def accepted_note_search_insert_statement(session: AsyncSession): def accepted_note_search_insert_params( row: AcceptedNoteSearchRow, + *, + include_full_content_grams: bool, ) -> SearchIndexSqlParams: """Build SQL parameters for one accepted-note search row.""" + script_texts = (row.title, row.content_stems) + if include_full_content_grams: + script_texts = (*script_texts, row.content_snippet) return { "id": row.id, "title": row.title, "content_stems": row.content_stems, "content_snippet": row.content_snippet, + "script_ngrams": build_script_ngrams(*script_texts), "permalink": row.permalink, "file_path": row.file_path, "type": row.item_type, @@ -134,10 +181,42 @@ async def refresh_entity( DELETE_ACCEPTED_NOTE_SEARCH_SQL, {"entity_id": row.entity_id, "project_id": row.project_id}, ) + is_sqlite = session.get_bind().dialect.name == "sqlite" await session.execute( accepted_note_search_insert_statement(session), - accepted_note_search_insert_params(row), + accepted_note_search_insert_params( + row, + include_full_content_grams=is_sqlite, + ), ) + if is_sqlite: + return + + # An upsert can move an older permalink owner's chunks through ON UPDATE CASCADE. + # Clear the final parent key before installing this accepted note's replacements. + await session.execute( + DELETE_ACCEPTED_NOTE_FTS_CHUNKS_SQL, + { + "project_id": row.project_id, + "search_index_id": row.id, + "search_index_type": row.item_type, + }, + ) + chunks = [ + { + "search_index_id": row.id, + "search_index_type": row.item_type, + "chunk_index": chunk_index, + "chunk_text": chunk_text, + "script_ngrams": build_script_ngrams(chunk_text), + } + for chunk_index, chunk_text in split_postgres_fts_chunks(row.content_snippet) + ] + if chunks: + await session.execute( + INSERT_ACCEPTED_NOTE_FTS_CHUNKS_SQL, + {"project_id": row.project_id, "chunks": json.dumps(chunks)}, + ) async def delete_entity( self, diff --git a/src/basic_memory/repository/postgres_fts_chunks.py b/src/basic_memory/repository/postgres_fts_chunks.py new file mode 100644 index 000000000..7cea623e2 --- /dev/null +++ b/src/basic_memory/repository/postgres_fts_chunks.py @@ -0,0 +1,19 @@ +"""Bounded PostgreSQL full-text search chunks.""" + +POSTGRES_FTS_CHUNK_SIZE = 8_000 +# PostgreSQL ignores lexemes at 2 KiB and above. A 2,048-character overlap is +# therefore conservative for every indexable lexeme, including multi-byte text: +# any token split at one 8,000-character edge is complete in the next chunk. +POSTGRES_FTS_CHUNK_OVERLAP = 2_048 + + +def split_postgres_fts_chunks(content: str | None) -> list[tuple[int, str]]: + """Split full note text without losing an indexable lexeme at a chunk edge.""" + if not content: + return [] + + step = POSTGRES_FTS_CHUNK_SIZE - POSTGRES_FTS_CHUNK_OVERLAP + return [ + (chunk_index, content[start : start + POSTGRES_FTS_CHUNK_SIZE]) + for chunk_index, start in enumerate(range(0, len(content), step)) + ] diff --git a/src/basic_memory/repository/postgres_search_repository.py b/src/basic_memory/repository/postgres_search_repository.py index e90c75d70..363f4bc74 100644 --- a/src/basic_memory/repository/postgres_search_repository.py +++ b/src/basic_memory/repository/postgres_search_repository.py @@ -21,6 +21,7 @@ from basic_memory.repository.rerank_provider_factory import create_rerank_provider from basic_memory.repository.search_index_row import SearchIndexRow from basic_memory.repository.search_query import relaxed_query_words, relaxation_word_tokens +from basic_memory.repository.script_ngrams import analyze_script_query, build_script_ngrams from basic_memory.repository.semantic_chunking import VectorChunkRecord from basic_memory.repository.search_repository_base import ( SearchRepositoryBase, @@ -42,14 +43,10 @@ resolve_semantic_vector_index_name, ) from basic_memory.repository.pgvector_index import PgVectorIndex +from basic_memory.repository.postgres_fts_chunks import split_postgres_fts_chunks from basic_memory.schemas.search import SearchItemType, SearchRetrievalMode -POSTGRES_FTS_CHUNK_SIZE = 8_000 -# PostgreSQL ignores lexemes at 2 KiB and above. A 2,048-character overlap is -# therefore conservative for every indexable lexeme, including multi-byte text: -# any token split at one 8,000-character edge is complete in the next chunk. -POSTGRES_FTS_CHUNK_OVERLAP = 2_048 _TSQUERY_OPERAND_PATTERN = re.compile(r"'(?:''|[^'])*'(?::\*)?|[^\s&|!()]+") _TSQUERY_WORD_PATTERN = re.compile(r"[^\W_]+(?:'[^\W_]+)?", re.UNICODE) _QUOTED_QUERY_PATTERN = re.compile(r'"([^"]*)"') @@ -57,18 +54,6 @@ _TSQUERY_METACHARACTERS = frozenset("&|!:<>") -def _iter_fts_chunks(content: str | None) -> list[tuple[int, str]]: - """Split full note text without losing an indexable lexeme at a chunk edge.""" - if not content: - return [] - - step = POSTGRES_FTS_CHUNK_SIZE - POSTGRES_FTS_CHUNK_OVERLAP - return [ - (chunk_index, content[start : start + POSTGRES_FTS_CHUNK_SIZE]) - for chunk_index, start in enumerate(range(0, len(content), step)) - ] - - def _tsquery_operands(processed_text: str) -> list[tuple[str, str]]: """Return unique (query operand, representative text) pairs in source order.""" operands: dict[str, str] = {} @@ -282,6 +267,10 @@ async def index_item(self, search_index_row: SearchIndexRow) -> None: # Serialize JSON for raw SQL insert_data = search_index_row.to_insert(serialize_json=True) insert_data["project_id"] = self.project_id + insert_data["script_ngrams"] = build_script_ngrams( + search_index_row.title, + search_index_row.content_stems, + ) insert_data = _strip_nul_from_row(insert_data) # Use upsert to handle race conditions during parallel indexing @@ -291,13 +280,13 @@ async def index_item(self, search_index_row: SearchIndexRow) -> None: await session.execute( text(""" INSERT INTO search_index ( - id, title, content_stems, content_snippet, permalink, file_path, type, metadata, + id, title, content_stems, content_snippet, script_ngrams, permalink, file_path, type, metadata, from_id, to_id, relation_type, entity_id, category, created_at, updated_at, project_id ) VALUES ( - :id, :title, :content_stems, :content_snippet, :permalink, :file_path, :type, :metadata, + :id, :title, :content_stems, :content_snippet, :script_ngrams, :permalink, :file_path, :type, :metadata, :from_id, :to_id, :relation_type, :entity_id, :category, :created_at, :updated_at, @@ -308,6 +297,7 @@ async def index_item(self, search_index_row: SearchIndexRow) -> None: title = EXCLUDED.title, content_stems = EXCLUDED.content_stems, content_snippet = EXCLUDED.content_snippet, + script_ngrams = EXCLUDED.script_ngrams, file_path = EXCLUDED.file_path, type = EXCLUDED.type, metadata = EXCLUDED.metadata, @@ -359,9 +349,10 @@ async def _replace_fts_chunks( "search_index_type": row.type, "chunk_index": chunk_index, "chunk_text": chunk_text.replace("\x00", ""), + "script_ngrams": build_script_ngrams(chunk_text.replace("\x00", "")), } for row in search_index_rows - for chunk_index, chunk_text in _iter_fts_chunks(row.content_snippet) + for chunk_index, chunk_text in split_postgres_fts_chunks(row.content_snippet) ] if not chunks: return @@ -373,19 +364,22 @@ async def _replace_fts_chunks( search_index_id, search_index_type, chunk_index, - chunk_text + chunk_text, + script_ngrams ) SELECT :project_id, chunk.search_index_id, chunk.search_index_type, chunk.chunk_index, - chunk.chunk_text + chunk.chunk_text, + chunk.script_ngrams FROM jsonb_to_recordset(CAST(:chunks AS JSONB)) AS chunk( search_index_id INTEGER, search_index_type VARCHAR, chunk_index INTEGER, - chunk_text TEXT + chunk_text TEXT, + script_ngrams TEXT ) """), {"project_id": self.project_id, "chunks": json.dumps(chunks)}, @@ -901,6 +895,10 @@ async def bulk_index_items(self, search_index_rows: List[SearchIndexRow]) -> Non for row in search_index_rows: insert_data = row.to_insert(serialize_json=True) insert_data["project_id"] = self.project_id + insert_data["script_ngrams"] = build_script_ngrams( + row.title, + row.content_stems, + ) insert_data_list.append(_strip_nul_from_row(insert_data)) # Use upsert to handle race conditions during parallel indexing @@ -910,13 +908,13 @@ async def bulk_index_items(self, search_index_rows: List[SearchIndexRow]) -> Non await session.execute( text(""" INSERT INTO search_index ( - id, title, content_stems, content_snippet, permalink, file_path, type, metadata, + id, title, content_stems, content_snippet, script_ngrams, permalink, file_path, type, metadata, from_id, to_id, relation_type, entity_id, category, created_at, updated_at, project_id ) VALUES ( - :id, :title, :content_stems, :content_snippet, :permalink, :file_path, :type, :metadata, + :id, :title, :content_stems, :content_snippet, :script_ngrams, :permalink, :file_path, :type, :metadata, :from_id, :to_id, :relation_type, :entity_id, :category, :created_at, :updated_at, @@ -927,6 +925,7 @@ async def bulk_index_items(self, search_index_rows: List[SearchIndexRow]) -> Non title = EXCLUDED.title, content_stems = EXCLUDED.content_stems, content_snippet = EXCLUDED.content_snippet, + script_ngrams = EXCLUDED.script_ngrams, file_path = EXCLUDED.file_path, type = EXCLUDED.type, metadata = EXCLUDED.metadata, @@ -977,6 +976,7 @@ async def _build_fts_query_parts( order_by_clause = "" from_clause = "search_index" document_vector_sql: str | None = None + script_tsqueries: list[str] = [] # Handle text search for title and content using tsvector if search_text: @@ -984,29 +984,30 @@ async def _build_fts_query_parts( # For wildcard searches, don't add any text conditions pass else: - # Prepare search term for tsquery - processed_text = self._prepare_search_term(search_text.strip()) - params["text"] = processed_text - probe_texts = [processed_text] - if allow_relaxed: - relaxed_text = self._relaxed_tsquery_text(search_text) - if relaxed_text: - probe_texts.append(relaxed_text) - - candidate_operands: dict[str, None] = {} - for probe_text in probe_texts: - for operand, _representative in _tsquery_operands(probe_text): - candidate_operands.setdefault(operand, None) - if candidate_operands: - params["text_candidate"] = " | ".join(candidate_operands) - - # Trigger: PostgreSQL can extract a required-positive query tree. - # Why: OR-ing its operands is a safe indexed superset even when - # terms live in different chunks. Pure/optional negation returns - # ``T`` and must retain all project rows for correct semantics. - # Outcome: ordinary and required-positive NOT queries use both - # GIN indexes; only genuinely unindexable negation scans the project. - from_clause = """ + script_query = analyze_script_query(search_text.strip()) + if script_query.word_text: + processed_text = self._prepare_search_term(script_query.word_text) + params["text"] = processed_text + probe_texts = [processed_text] + if allow_relaxed: + relaxed_text = self._relaxed_tsquery_text(script_query.word_text) + if relaxed_text: + probe_texts.append(relaxed_text) + + candidate_operands: dict[str, None] = {} + for probe_text in probe_texts: + for operand, _representative in _tsquery_operands(probe_text): + candidate_operands.setdefault(operand, None) + if candidate_operands: + params["text_candidate"] = " | ".join(candidate_operands) + + # Trigger: PostgreSQL can extract a required-positive query tree. + # Why: OR-ing its operands is a safe indexed superset even when + # terms live in different chunks. Pure/optional negation returns + # ``T`` and must retain all project rows for correct semantics. + # Outcome: ordinary and required-positive NOT queries use both + # GIN indexes; only genuinely unindexable negation scans the project. + from_clause = """ search_index JOIN ( SELECT candidate_parent.project_id, @@ -1039,9 +1040,72 @@ async def _build_fts_query_parts( ON fts_candidate.project_id = search_index.project_id AND fts_candidate.id = search_index.id AND fts_candidate.type = search_index.type + """ + document_vector_sql = self._document_fts_vector_sql(probe_texts, params) + word_condition = f"{document_vector_sql} @@ to_tsquery('english', :text)" + if script_query.gram_phrases: + # Trigger: PostgreSQL's English dictionary removes every word term. + # Why: an empty word query must not suppress a required script match. + # Outcome: only mixed queries treat the empty word channel as neutral; + # word-only stopword queries retain their established empty result. + word_condition = ( + f"(numnode(to_tsquery('english', :text)) = 0 OR {word_condition})" + ) + conditions.append(word_condition) + + if script_query.gram_phrases: + script_tsqueries = [ + " <-> ".join( + f"'{gram}':*" if gram.startswith("bmprefix") else f"'{gram}'" + for gram in phrase + ) + for phrase in script_query.gram_phrases + ] + for index, script_tsquery in enumerate(script_tsqueries): + params[f"script_text_{index}"] = script_tsquery + # Trigger: a query contains script grams, with or without word terms. + # Why: every script phrase is required, while an English word clause can + # reduce to an empty tsquery after dictionary processing. + # Outcome: start from the parent and child script GIN indexes, then apply + # every word and script predicate below. + params["script_candidate_text"] = " | ".join( + f"({script_tsquery})" for script_tsquery in script_tsqueries + ) + from_clause = """ + search_index JOIN ( + SELECT + script_parent.project_id, + script_parent.id, + script_parent.type + FROM search_index AS script_parent + WHERE script_parent.project_id = :project_id + AND script_parent.script_ngrams_index_col + @@ to_tsquery('simple', :script_candidate_text) + UNION + SELECT + script_candidate.project_id, + script_candidate.search_index_id AS id, + script_candidate.search_index_type AS type + FROM search_index_fts_chunks AS script_candidate + WHERE script_candidate.project_id = :project_id + AND script_candidate.script_ngrams_index_col + @@ to_tsquery('simple', :script_candidate_text) + ) AS fts_candidate + ON fts_candidate.project_id = search_index.project_id + AND fts_candidate.id = search_index.id + AND fts_candidate.type = search_index.type """ - document_vector_sql = self._document_fts_vector_sql(probe_texts, params) - conditions.append(f"{document_vector_sql} @@ to_tsquery('english', :text)") + conditions.extend( + "(search_index.script_ngrams_index_col " + f"@@ to_tsquery('simple', :script_text_{index}) OR EXISTS (" + "SELECT 1 FROM search_index_fts_chunks AS script_chunk " + "WHERE script_chunk.project_id = search_index.project_id " + "AND script_chunk.search_index_id = search_index.id " + "AND script_chunk.search_index_type = search_index.type " + "AND script_chunk.script_ngrams_index_col " + f"@@ to_tsquery('simple', :script_text_{index})))" + for index in range(len(script_tsqueries)) + ) # Handle title search if title: @@ -1199,9 +1263,9 @@ async def _build_fts_query_parts( # Build SQL with ts_rank() for scoring # Note: If no text search, score will be NULL, so we use COALESCE to default to 0 - if search_text and search_text.strip() and search_text.strip() != "*": - assert document_vector_sql is not None - score_expr = ( + score_parts: list[str] = [] + if document_vector_sql is not None: + score_parts.append( "GREATEST(" f"ts_rank({document_vector_sql}, to_tsquery('english', :text)), " "ts_rank(search_index.textsearchable_index_col, to_tsquery('english', :text)), " @@ -1214,8 +1278,23 @@ async def _build_fts_query_parts( "AND fts_chunk.textsearchable_index_col " "@@ to_tsquery('english', :text)), 0))" ) - else: - score_expr = "0" + score_parts.extend( + "GREATEST(" + "ts_rank(search_index.script_ngrams_index_col, " + f"to_tsquery('simple', :script_text_{index})), " + "COALESCE((SELECT MAX(ts_rank(script_rank.script_ngrams_index_col, " + f"to_tsquery('simple', :script_text_{index}))) " + "FROM search_index_fts_chunks AS script_rank " + "WHERE script_rank.project_id = search_index.project_id " + "AND script_rank.search_index_id = search_index.id " + "AND script_rank.search_index_type = search_index.type " + "AND script_rank.script_ngrams_index_col " + f"@@ to_tsquery('simple', :script_text_{index})), 0))" + for index in range(len(script_tsqueries)) + ) + # Each condition above is required, so every query component should contribute to + # relevance. Taking only the strongest rank makes additional script runs invisible. + score_expr = " + ".join(score_parts) if score_parts else "0" return from_clause, where_clause, params, order_by_clause, score_expr diff --git a/src/basic_memory/repository/script_ngrams.py b/src/basic_memory/repository/script_ngrams.py new file mode 100644 index 000000000..c993c1228 --- /dev/null +++ b/src/basic_memory/repository/script_ngrams.py @@ -0,0 +1,223 @@ +"""Application-owned lexical analysis for scripts without reliable word boundaries.""" + +import hashlib +import unicodedata +from dataclasses import dataclass + + +_SCRIPT_BOUNDARY = "bm_script_boundary" +MIXED_WORD_BLOCK_BYTES = 24 + + +@dataclass(frozen=True, slots=True) +class ScriptQuery: + """The word-oriented and script-oriented parts of one user query.""" + + word_text: str | None + gram_phrases: tuple[tuple[str, ...], ...] + + +def is_script_search_character(character: str) -> bool: + codepoint = ord(character) + return any( + lower <= codepoint <= upper + for lower, upper in ( + (0x1100, 0x11FF), # Hangul Jamo + (0x0E00, 0x0EFF), # Thai and Lao + (0x0F00, 0x0FFF), # Tibetan + (0x1000, 0x109F), # Myanmar + (0x1780, 0x17FF), # Khmer + (0x19E0, 0x19FF), # Khmer symbols + (0x3005, 0x3007), # Ideographic iteration, closing, and zero marks + (0x3021, 0x3029), # Hangzhou numerals + (0x3031, 0x3035), # Vertical kana repeat marks + (0x3038, 0x303B), # Hangzhou tens and vertical iteration mark + (0x3040, 0x30FF), # Hiragana and Katakana + (0x3100, 0x318F), # Bopomofo and Hangul compatibility Jamo + (0x31A0, 0x31BF), # Bopomofo extended + (0x31F0, 0x31FF), # Katakana phonetic extensions + (0x3400, 0x4DBF), # CJK unified ideographs extension A + (0x4E00, 0x9FFF), # CJK unified ideographs + (0xA960, 0xA97F), # Hangul Jamo extended A + (0xA9E0, 0xA9FF), # Myanmar extended B + (0xAA60, 0xAA7F), # Myanmar extended A + (0xAC00, 0xD7AF), # Hangul syllables + (0xD7B0, 0xD7FF), # Hangul Jamo extended B + (0xF900, 0xFAFF), # CJK compatibility ideographs + (0x1AFF0, 0x1AFFF), # Katakana extended B + (0x1B000, 0x1B16F), # Kana supplements and extensions + (0x20000, 0x2FFFF), # Supplementary CJK ideographs + (0x30000, 0x323AF), # CJK unified ideographs extensions G-H + ) + ) + + +def script_runs(text: str) -> tuple[tuple[str, ...], ...]: + """Return normalized script runs as grapheme-like searchable units.""" + runs: list[tuple[str, ...]] = [] + current: list[str] = [] + for character in unicodedata.normalize("NFKC", text): + # Join controls shape neighboring script characters without introducing a searchable + # unit. Keeping the current run open preserves ordered matching across the control. + if character in {"\u200c", "\u200d"}: + continue + if current and unicodedata.category(character) in {"Mn", "Mc", "Me"}: + current[-1] += character + continue + if is_script_search_character(character): + current.append(character) + continue + if current: + runs.append(tuple(current)) + current = [] + if current: + runs.append(tuple(current)) + return tuple(runs) + + +def script_run_grams(run: tuple[str, ...]) -> tuple[str, ...]: + """Use bigrams for context and retain a searchable single-character run.""" + if len(run) == 1: + return run + return tuple(first + second for first, second in zip(run, run[1:], strict=False)) + + +def mixed_token_word_terms(text: str) -> tuple[str, ...]: + """Encode prefix-searchable word fragments and their order around script runs.""" + terms: list[str] = [] + for token in text.split(): + normalized_token = unicodedata.normalize("NFKC", token) + if not any(is_script_search_character(character) for character in normalized_token): + continue + + components: list[tuple[str, str]] = [] + component_kind: str | None = None + component_characters: list[str] = [] + for character in normalized_token: + if character in {"\u200c", "\u200d"}: + continue + if component_characters and unicodedata.category(character) in {"Mn", "Mc", "Me"}: + component_characters.append(character) + continue + + next_kind = ( + "script" + if is_script_search_character(character) + else "word" + if character.isalnum() + else None + ) + if next_kind != component_kind and component_characters: + components.append((component_kind or "word", "".join(component_characters))) + component_characters = [] + component_kind = next_kind + if next_kind is not None: + component_characters.append(character) + if component_characters: + components.append((component_kind or "word", "".join(component_characters))) + + word_roles: dict[int, list[tuple[str, int]]] = {} + after_distance = 0 + has_script_before = False + for index, (kind, _) in enumerate(components): + if kind == "script": + after_distance = 0 + has_script_before = True + continue + if not has_script_before: + continue + after_distance += 1 + word_roles.setdefault(index, []).append(("after", after_distance)) + + before_distance = 0 + has_script_after = False + for index in range(len(components) - 1, -1, -1): + kind, _ = components[index] + if kind == "script": + before_distance = 0 + has_script_after = True + continue + if not has_script_after: + continue + before_distance += 1 + word_roles.setdefault(index, []).append(("before", before_distance)) + + # Fixed-size blocks preserve arbitrary-length prefix matching while keeping every + # generated lexeme and the total auxiliary representation linear in the input size. + for index, roles in word_roles.items(): + word_bytes = components[index][1].casefold().encode() + for direction, distance in roles: + terms.extend( + f"bmprefix{direction}{distance}x{block_index}x" + f"{word_bytes[start : start + MIXED_WORD_BLOCK_BYTES].hex()}" + for block_index, start in enumerate( + range(0, len(word_bytes), MIXED_WORD_BLOCK_BYTES) + ) + ) + + # Role terms bind the positional word blocks to their neighboring script component. + # A separate token cannot satisfy these terms merely by containing the same script gram. + for index, (kind, value) in enumerate(components): + if kind != "script": + continue + run = tuple(unit for script_run in script_runs(value) for unit in script_run) + run_terms = (*run, *script_run_grams(run)) + if index > 0 and components[index - 1][0] == "word": + terms.extend( + "bmrolebefore" + hashlib.sha256(term.encode()).hexdigest() for term in run_terms + ) + if index + 1 < len(components) and components[index + 1][0] == "word": + terms.extend( + "bmroleafter" + hashlib.sha256(term.encode()).hexdigest() for term in run_terms + ) + return tuple(dict.fromkeys(terms)) + + +def build_script_ngrams(*texts: str | None) -> str: + """Build portable index text without depending on a database tokenizer.""" + gram_runs: list[str] = [] + for text in texts: + if not text: + continue + for run in script_runs(text): + # Unigrams make a single-character query searchable inside a longer run. Keeping + # bigrams together after them preserves ordered phrase matching for longer queries. + index_terms = run if len(run) == 1 else (*run, *script_run_grams(run)) + gram_runs.append(" ".join(index_terms)) + gram_runs.extend(mixed_token_word_terms(text)) + return f" {_SCRIPT_BOUNDARY} ".join(gram_runs) + + +def analyze_script_query(text: str) -> ScriptQuery: + """Split a natural-language query into word text and ordered script grams.""" + normalized = unicodedata.normalize("NFKC", text) + # Explicit Boolean expressions keep the existing backend parser. Splitting one + # operand into a second SQL channel would otherwise change OR/NOT semantics. + # Compatibility forms are natural-language text because neither backend parses + # their normalized equivalents as operators. + padded_text = f" {text} " + if '"' in text or any(f" {operator} " in padded_text for operator in ("AND", "OR", "NOT")): + return ScriptQuery(word_text=text, gram_phrases=()) + + # Mixed tokens use the same application-owned auxiliary channel as script grams. This + # avoids assuming that either backend exposes word fragments on both sides of a script run. + word_tokens = [ + token.lower() if token in {"AND", "OR", "NOT"} else token + for token in text.split() + if not any( + is_script_search_character(character) + for character in unicodedata.normalize("NFKC", token) + ) + and any(character.isalnum() for character in unicodedata.normalize("NFKC", token)) + ] + gram_phrases = ( + *(script_run_grams(run) for run in script_runs(normalized)), + *((term,) for term in mixed_token_word_terms(text)), + ) + # Preserve punctuation-only input as an explicit backend query. Dropping it would make + # the repositories confuse user text with the intentional no-predicate wildcard path. + word_text = " ".join(word_tokens) or (text if not gram_phrases else None) + return ScriptQuery( + word_text=word_text, + gram_phrases=gram_phrases, + ) diff --git a/src/basic_memory/repository/search_repository_base.py b/src/basic_memory/repository/search_repository_base.py index 81ed70c9f..9b6d7e205 100644 --- a/src/basic_memory/repository/search_repository_base.py +++ b/src/basic_memory/repository/search_repository_base.py @@ -32,6 +32,7 @@ validate_rerank_scores, ) from basic_memory.repository.search_index_row import SearchIndexRow +from basic_memory.repository.script_ngrams import build_script_ngrams from basic_memory.repository.search_trace import ( BelowThreshold, FilteredOut, @@ -993,18 +994,23 @@ async def index_item(self, search_index_row: SearchIndexRow) -> None: # The database driver/column type will handle conversion insert_data = search_index_row.to_insert(serialize_json=True) insert_data["project_id"] = self.project_id + insert_data["script_ngrams"] = build_script_ngrams( + search_index_row.title, + search_index_row.content_stems, + search_index_row.content_snippet, + ) # Insert new record await session.execute( text(""" INSERT INTO search_index ( - id, title, content_stems, content_snippet, permalink, file_path, type, metadata, + id, title, content_stems, content_snippet, script_ngrams, permalink, file_path, type, metadata, from_id, to_id, relation_type, entity_id, category, created_at, updated_at, project_id ) VALUES ( - :id, :title, :content_stems, :content_snippet, :permalink, :file_path, :type, :metadata, + :id, :title, :content_stems, :content_snippet, :script_ngrams, :permalink, :file_path, :type, :metadata, :from_id, :to_id, :relation_type, :entity_id, :category, :created_at, :updated_at, @@ -1039,19 +1045,24 @@ async def bulk_index_items(self, search_index_rows: List[SearchIndexRow]) -> Non for row in search_index_rows: insert_data = row.to_insert(serialize_json=True) insert_data["project_id"] = self.project_id + insert_data["script_ngrams"] = build_script_ngrams( + row.title, + row.content_stems, + row.content_snippet, + ) insert_data_list.append(insert_data) # Batch insert all records using executemany await session.execute( text(""" INSERT INTO search_index ( - id, title, content_stems, content_snippet, permalink, file_path, type, metadata, + id, title, content_stems, content_snippet, script_ngrams, permalink, file_path, type, metadata, from_id, to_id, relation_type, entity_id, category, created_at, updated_at, project_id ) VALUES ( - :id, :title, :content_stems, :content_snippet, :permalink, :file_path, :type, :metadata, + :id, :title, :content_stems, :content_snippet, :script_ngrams, :permalink, :file_path, :type, :metadata, :from_id, :to_id, :relation_type, :entity_id, :category, :created_at, :updated_at, diff --git a/src/basic_memory/repository/sqlite_search_repository.py b/src/basic_memory/repository/sqlite_search_repository.py index 9dced81f9..757679635 100644 --- a/src/basic_memory/repository/sqlite_search_repository.py +++ b/src/basic_memory/repository/sqlite_search_repository.py @@ -29,6 +29,7 @@ from basic_memory.repository.search_index_row import SearchIndexRow from basic_memory.repository.search_query import relaxed_query_words from basic_memory.repository.search_repository_base import SearchRepositoryBase +from basic_memory.repository.script_ngrams import analyze_script_query from basic_memory.repository.search_trace import ( SearchTraceCollector, build_fts_page_stage, @@ -42,6 +43,9 @@ from basic_memory.schemas.search import SearchItemType, SearchRetrievalMode +SQLITE_WORD_COLUMNS = "{title content_stems content_snippet}" + + class SQLiteSearchRepository(SearchRepositoryBase): """SQLite FTS5 implementation of search repository. @@ -778,13 +782,15 @@ async def _build_fts_query_parts( search_item_types: Optional[List[SearchItemType]] = None, categories: Optional[List[str]] = None, metadata_filters: Optional[dict[str, Any]] = None, - ) -> tuple[str, str, dict[str, Any], str]: + ) -> tuple[str, str, dict[str, Any], str, str]: """Build SQLite FTS FROM/WHERE params shared by search and count.""" conditions = [] match_conditions = [] params = {} order_by_clause = "" from_clause = "search_index" + score_expression = "bm25(search_index)" + preserve_match_score = False # Handle text search for title and content if search_text: @@ -793,15 +799,48 @@ async def _build_fts_query_parts( # For wildcard searches, don't add any text conditions - return all results pass else: - # Use _prepare_search_term to handle both Boolean and non-Boolean queries - processed_text = self._prepare_search_term(search_text.strip()) - params["text"] = processed_text - # content_stems is capped for Postgres index-row compatibility, while - # SQLite stores the complete note body in its FTS5 content_snippet column. - match_conditions.append( - "(search_index.title MATCH :text OR search_index.content_stems MATCH :text " - "OR search_index.content_snippet MATCH :text)" - ) + script_query = analyze_script_query(search_text.strip()) + # Trigger: the query contains text from an unsegmented script. + # Why: the script channel needs one table-level MATCH alongside word fields. + # Outcome: mixed queries rank all terms together; word-only queries retain their + # established per-column matching and ranking behavior. + if script_query.gram_phrases: + preserve_match_score = True + params["text"] = "" + params["script_text"] = "" + if script_query.word_text: + prepared_text = self._prepare_search_term(script_query.word_text) + params["text"] = ( + f"(title: ({prepared_text}) OR " + f"content_stems: ({prepared_text}) OR " + f"content_snippet: ({prepared_text}))" + ) + script_phrases = " AND ".join( + f'"{" ".join(phrase)}"*' + if len(phrase) == 1 and phrase[0].startswith("bmprefix") + else f'"{" ".join(phrase)}"' + for phrase in script_query.gram_phrases + ) + script_clause = f"script_ngrams: ({script_phrases})" + params["script_text"] = ( + f" AND ({script_clause})" if script_query.word_text else script_clause + ) + match_conditions.append("search_index MATCH (:text || :script_text)") + else: + word_text = ( + script_query.word_text + if script_query.word_text is not None + else search_text.strip() + ) + processed_text = self._prepare_search_term(word_text) + params["text"] = processed_text + # content_stems is capped for Postgres index-row compatibility, while + # SQLite stores the complete note body in its FTS5 content_snippet column. + match_conditions.append( + "(search_index.title MATCH :text OR " + "search_index.content_stems MATCH :text OR " + "search_index.content_snippet MATCH :text)" + ) # Handle title match search if title: @@ -964,15 +1003,38 @@ async def _build_fts_query_parts( conditions.append(f"{compare_expr} {operator} :{value_param}") continue + # Trigger: SQLite rejects some Boolean combinations of MATCH predicates, + # including a word-field OR expression combined with the script channel. + # Why: each MATCH must be evaluated in an FTS-valid query context. + # Outcome: keep one outer MATCH for bm25 ranking and intersect the rest by rowid. + if len(match_conditions) > 1: + ranked_match, *additional_matches = match_conditions + conditions.extend( + f"search_index.rowid IN (SELECT rowid FROM search_index WHERE {match_condition})" + for match_condition in additional_matches + ) + match_conditions = [ranked_match] + # Trigger: SQLite FTS MATCH predicates combined with JOINs can fail with # "unable to use function MATCH in the requested context". - # Why: MATCH needs to run in an FTS-valid context. - # Outcome: evaluate MATCH clauses in an FTS subquery and filter outer rows by rowid. + # Why: script queries need MATCH and bm25 together for ranking, while legacy + # word-column OR predicates cannot evaluate bm25 in the same derived query. + # Outcome: rank script matches before joining metadata; retain the established + # rowid-filter path for word-only searches. if metadata_filters and match_conditions: match_where = " AND ".join(match_conditions) - conditions.append( - f"search_index.rowid IN (SELECT rowid FROM search_index WHERE {match_where})" - ) + if preserve_match_score: + from_clause = ( + "(SELECT search_index.rowid AS rowid, search_index.*, " + "bm25(search_index) AS fts_score " + f"FROM search_index WHERE {match_where}) AS search_index " + "JOIN entity ON search_index.entity_id = entity.id" + ) + score_expression = "search_index.fts_score" + else: + conditions.append( + f"search_index.rowid IN (SELECT rowid FROM search_index WHERE {match_where})" + ) else: conditions.extend(match_conditions) @@ -982,7 +1044,7 @@ async def _build_fts_query_parts( # Build WHERE clause where_clause = " AND ".join(conditions) if conditions else "1=1" - return from_clause, where_clause, params, order_by_clause + return from_clause, where_clause, params, order_by_clause, score_expression @override async def search( @@ -1033,7 +1095,13 @@ async def search( return dispatched # --- FTS mode (SQLite-specific) --- - from_clause, where_clause, params, order_by_clause = await self._build_fts_query_parts( + ( + from_clause, + where_clause, + params, + order_by_clause, + score_expression, + ) = await self._build_fts_query_parts( search_text=search_text, permalink=permalink, permalink_match=permalink_match, @@ -1048,6 +1116,9 @@ async def search( # set limit on search query params["limit"] = limit params["offset"] = offset + relaxed_search_text = search_text + if search_text and "script_text" in params: + relaxed_search_text = analyze_script_query(search_text.strip()).word_text sql = f""" SELECT @@ -1066,7 +1137,7 @@ async def search( search_index.category, search_index.created_at, search_index.updated_at, - bm25(search_index) as score + {score_expression} as score FROM {from_clause} WHERE {where_clause} ORDER BY score ASC {order_by_clause} @@ -1089,10 +1160,14 @@ async def run_search(active_session: AsyncSession): # vector-only. # Outcome: one retry with OR-joined prefix terms; bm25 still # ranks multi-term matches first. - relaxed = self._relaxed_fts_text(search_text) if allow_relaxed and not rows else None + relaxed = ( + self._relaxed_fts_text(relaxed_search_text) if allow_relaxed and not rows else None + ) if relaxed and params.get("text"): relaxed_fallback_used = True - params["text"] = relaxed + params["text"] = ( + f"{SQLITE_WORD_COLUMNS}: ({relaxed})" if "script_text" in params else relaxed + ) logger.debug( "Strict SQLite FTS returned 0 results; retrying relaxed FTS query " f"strict='{search_text}' relaxed='{relaxed}'" @@ -1100,7 +1175,7 @@ async def run_search(active_session: AsyncSession): with logfire.span( "search.relaxed_fts_retry", backend="sqlite", - token_count=len(relaxed_query_words(search_text) or ()), + token_count=len(relaxed_query_words(relaxed_search_text) or ()), limit=limit, offset=offset, ): @@ -1187,7 +1262,13 @@ async def count( min_similarity=min_similarity, ) - from_clause, where_clause, params, _order_by_clause = await self._build_fts_query_parts( + ( + from_clause, + where_clause, + params, + _order_by_clause, + _score_expression, + ) = await self._build_fts_query_parts( search_text=search_text, permalink=permalink, permalink_match=permalink_match, @@ -1200,19 +1281,28 @@ async def count( ) sql = f"SELECT COUNT(*) FROM {from_clause} WHERE {where_clause}" logger.trace(f"Count {sql} params: {params}") + relaxed_search_text = search_text + if search_text and "script_text" in params: + relaxed_search_text = analyze_script_query(search_text.strip()).word_text try: async with db.scoped_session(self.session_maker) as session: result = await session.execute(text(sql), params) total = int(result.scalar_one()) relaxed = ( - self._relaxed_fts_text(search_text) if allow_relaxed and total == 0 else None + self._relaxed_fts_text(relaxed_search_text) + if allow_relaxed and total == 0 + else None ) if relaxed and params.get("text"): - params["text"] = relaxed + params["text"] = ( + f"{SQLITE_WORD_COLUMNS}: ({relaxed})" + if "script_text" in params + else relaxed + ) with logfire.span( "search.count.relaxed_fts_retry", backend="sqlite", - token_count=len(relaxed_query_words(search_text) or ()), + token_count=len(relaxed_query_words(relaxed_search_text) or ()), ): result = await session.execute(text(sql), params) total = int(result.scalar_one()) diff --git a/tests/conftest.py b/tests/conftest.py index ef6cdb9d1..3769bfd2b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -153,7 +153,9 @@ async def _reset_postgres_test_schema(engine: AsyncEngine, async_url: str) -> No from basic_memory.models.search import ( CREATE_POSTGRES_SEARCH_INDEX_FTS, CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_INDEX, + CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_SCRIPT_NGRAMS_INDEX, CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_TABLE, + CREATE_POSTGRES_SEARCH_INDEX_SCRIPT_NGRAMS_FTS, CREATE_POSTGRES_SEARCH_INDEX_METADATA, CREATE_POSTGRES_SEARCH_INDEX_PERMALINK, CREATE_POSTGRES_SEARCH_INDEX_TABLE, @@ -168,8 +170,10 @@ async def _reset_postgres_test_schema(engine: AsyncEngine, async_url: str) -> No await conn.run_sync(Base.metadata.create_all) await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_TABLE) await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_FTS) + await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_SCRIPT_NGRAMS_FTS) await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_TABLE) await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_INDEX) + await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_SCRIPT_NGRAMS_INDEX) await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_METADATA) await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_PERMALINK) await conn.execute(CREATE_POSTGRES_SEARCH_VECTOR_CHUNKS_TABLE) diff --git a/tests/repository/test_accepted_note_search_repository.py b/tests/repository/test_accepted_note_search_repository.py index bbdf3bf0d..6341cf380 100644 --- a/tests/repository/test_accepted_note_search_repository.py +++ b/tests/repository/test_accepted_note_search_repository.py @@ -1,15 +1,19 @@ """Tests for accepted-note search repository operations.""" +import json from datetime import UTC, datetime from typing import Any, cast import pytest from sqlalchemy.ext.asyncio import AsyncSession +from basic_memory import db from basic_memory.indexing.accepted_note_search import build_accepted_note_search_row from basic_memory.repository.accepted_note_search_repository import ( AcceptedNoteSearchRepository, ) +from basic_memory.repository.script_ngrams import build_script_ngrams +from basic_memory.repository.postgres_search_repository import PostgresSearchRepository class _Dialect: @@ -47,7 +51,7 @@ async def test_refresh_entity_replaces_project_scoped_hot_search_row() -> None: entity_metadata={"tags": ["strategy"]}, permalink="main/project-plan", file_path="notes/project-plan.md", - search_content="Main body", + search_content="Main body 适者生存", created_at=created_at, updated_at=updated_at, project_id=7, @@ -55,18 +59,20 @@ async def test_refresh_entity_replaces_project_scoped_hot_search_row() -> None: await repository.refresh_entity(cast(AsyncSession, session), row) - assert len(session.executed) == 2 + assert len(session.executed) == 4 delete_sql, delete_params = session.executed[0] insert_sql, insert_params = session.executed[1] assert "DELETE FROM search_index" in delete_sql assert delete_params == {"entity_id": 42, "project_id": 7} assert "CAST(:metadata AS jsonb)" in insert_sql assert "ON CONFLICT (permalink, project_id)" in insert_sql + assert "script_ngrams = EXCLUDED.script_ngrams" in insert_sql assert insert_params == { "id": 42, "title": "Project Plan", "content_stems": row.content_stems, - "content_snippet": "Main body", + "content_snippet": "Main body 适者生存", + "script_ngrams": build_script_ngrams(row.title, row.content_stems), "permalink": "main/project-plan", "file_path": "notes/project-plan.md", "type": "entity", @@ -76,6 +82,25 @@ async def test_refresh_entity_replaces_project_scoped_hot_search_row() -> None: "updated_at": updated_at, "project_id": 7, } + chunk_delete_sql, chunk_delete_params = session.executed[2] + assert "DELETE FROM search_index_fts_chunks" in chunk_delete_sql + assert chunk_delete_params == { + "project_id": 7, + "search_index_id": 42, + "search_index_type": "entity", + } + chunk_sql, chunk_params = session.executed[3] + assert "INSERT INTO search_index_fts_chunks" in chunk_sql + assert chunk_params["project_id"] == 7 + assert json.loads(chunk_params["chunks"]) == [ + { + "search_index_id": 42, + "search_index_type": "entity", + "chunk_index": 0, + "chunk_text": "Main body 适者生存", + "script_ngrams": build_script_ngrams("Main body 适者生存"), + } + ] @pytest.mark.asyncio @@ -102,6 +127,7 @@ async def test_refresh_entity_uses_plain_insert_for_sqlite_virtual_table() -> No assert "ON CONFLICT" not in insert_sql assert "CAST(:metadata AS jsonb)" not in insert_sql assert ":metadata" in insert_sql + assert ":script_ngrams" in insert_sql @pytest.mark.asyncio @@ -126,3 +152,107 @@ async def test_refresh_entity_rejects_cross_project_rows() -> None: await repository.refresh_entity(cast(AsyncSession, session), row) assert session.executed == [] + + +@pytest.mark.asyncio +async def test_refresh_entity_is_immediately_searchable_by_script_substring( + search_repository, + session_maker, +) -> None: + repository = AcceptedNoteSearchRepository(project_id=search_repository.project_id) + now = datetime(2026, 6, 18, 12, 0, tzinfo=UTC) + row = build_accepted_note_search_row( + entity_id=42, + title="Evolution", + note_type="note", + entity_metadata=None, + permalink="main/evolution", + file_path="notes/evolution.md", + search_content="即适者生存的讨论", + created_at=now, + updated_at=now, + project_id=search_repository.project_id, + ) + + async with db.scoped_session(session_maker) as session: + await repository.refresh_entity(session, row) + + results = await search_repository.search("适者生存") + + assert [result.id for result in results] == [42] + + +@pytest.mark.asyncio +async def test_postgres_refresh_entity_chunks_large_script_content( + search_repository, + session_maker, +) -> None: + if not isinstance(search_repository, PostgresSearchRepository): + pytest.skip("PostgreSQL stores full note bodies in bounded FTS chunks") + + repository = AcceptedNoteSearchRepository(project_id=search_repository.project_id) + now = datetime(2026, 6, 18, 12, 0, tzinfo=UTC) + row = build_accepted_note_search_row( + entity_id=43, + title="Long evolution note", + note_type="note", + entity_metadata=None, + permalink="main/long-evolution", + file_path="notes/long-evolution.md", + search_content=f"{'進化' * 5_000}适者生存", + created_at=now, + updated_at=now, + project_id=search_repository.project_id, + ) + + async with db.scoped_session(session_maker) as session: + await repository.refresh_entity(session, row) + + results = await search_repository.search("适者生存") + + assert [result.id for result in results] == [43] + + +@pytest.mark.asyncio +async def test_postgres_refresh_entity_replaces_cascaded_permalink_chunks( + search_repository, + session_maker, +) -> None: + if not isinstance(search_repository, PostgresSearchRepository): + pytest.skip("PostgreSQL cascades chunk parent keys during permalink upserts") + + repository = AcceptedNoteSearchRepository(project_id=search_repository.project_id) + now = datetime(2026, 6, 18, 12, 0, tzinfo=UTC) + old_row = build_accepted_note_search_row( + entity_id=44, + title="Old owner", + note_type="note", + entity_metadata=None, + permalink="main/reassigned", + file_path="notes/old-owner.md", + search_content="旧所有者内容", + created_at=now, + updated_at=now, + project_id=search_repository.project_id, + ) + new_row = build_accepted_note_search_row( + entity_id=45, + title="New owner", + note_type="note", + entity_metadata=None, + permalink="main/reassigned", + file_path="notes/new-owner.md", + search_content="新所有者适者生存", + created_at=now, + updated_at=now, + project_id=search_repository.project_id, + ) + + async with db.scoped_session(session_maker) as session: + await repository.refresh_entity(session, old_row) + async with db.scoped_session(session_maker) as session: + await repository.refresh_entity(session, new_row) + + results = await search_repository.search("适者生存") + + assert [result.id for result in results] == [45] diff --git a/tests/repository/test_script_ngrams.py b/tests/repository/test_script_ngrams.py new file mode 100644 index 000000000..5e7f05769 --- /dev/null +++ b/tests/repository/test_script_ngrams.py @@ -0,0 +1,959 @@ +"""Portable script n-gram analysis and full-text search regressions.""" + +from datetime import datetime, timezone + +import pytest + +from basic_memory import db +from basic_memory.models import Entity +from basic_memory.repository.script_ngrams import ( + MIXED_WORD_BLOCK_BYTES, + analyze_script_query, + build_script_ngrams, + mixed_token_word_terms, + script_run_grams, + script_runs, +) +from basic_memory.repository.postgres_search_repository import PostgresSearchRepository +from basic_memory.repository.search_index_row import SearchIndexRow +from basic_memory.repository.sqlite_search_repository import SQLiteSearchRepository + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("适者生存", (("适", "者", "生", "存"),)), + ("適者生存", (("適", "者", "生", "存"),)), + ("サバイバル", (("サ", "バ", "イ", "バ", "ル"),)), + ("생존 경쟁", (("생", "존"), ("경", "쟁"))), + ("ภาษาไทย", (("ภ", "า", "ษ", "า", "ไ", "ท", "ย"),)), + ("時々", (("時", "々"),)), + ("\U0001aff0\U0001aff3\U0001affd", (("\U0001aff0", "\U0001aff3", "\U0001affd"),)), + ("ＡＢＣ", ()), + ], +) +def test_script_runs_cover_cjk_scripts_and_normalize_width( + text: str, + expected: tuple[tuple[str, ...], ...], +) -> None: + assert script_runs(text) == expected + + +def test_script_run_grams_preserve_order_and_single_character_queries() -> None: + assert script_run_grams(("适", "者", "生", "存")) == ("适者", "者生", "生存") + assert script_run_grams(("猫",)) == ("猫",) + + +def test_script_runs_attach_combining_marks_to_the_previous_unit() -> None: + text = "漢\N{VARIATION SELECTOR-1}" + + assert script_runs(text) == ((text,),) + assert analyze_script_query(text).word_text is None + + +@pytest.mark.parametrize("join_control", ["\u200c", "\u200d"]) +def test_script_runs_keep_join_controls_inside_ordered_runs(join_control: str) -> None: + text = f"ក{join_control}ខ" + + assert script_runs(text) == (("ក", "ខ"),) + assert analyze_script_query(text).gram_phrases == (("កខ",),) + + +def test_build_script_ngrams_keeps_runs_from_matching_across_boundaries() -> None: + assert build_script_ngrams("适者", "生存") == ("适 者 适者 bm_script_boundary 生 存 生存") + + +def test_mixed_token_word_terms_encode_all_word_fragments() -> None: + terms = mixed_token_word_terms("foo不適者bar ＡＢＣ適者") + + assert "bmprefixbefore1x0x666f6f" in terms + assert "bmprefixafter1x0x626172" in terms + assert "bmprefixbefore1x0x616263" in terms + assert any(term.startswith("bmrole") for term in terms) + + +def test_mixed_token_word_terms_bound_long_fragment_expansion() -> None: + terms = mixed_token_word_terms(f"{'a' * 500}{'漢字' * 250}") + + word_block_count = (500 + MIXED_WORD_BLOCK_BYTES - 1) // MIXED_WORD_BLOCK_BYTES + assert len(terms) <= word_block_count + 4 + + +def test_analyze_script_query_separates_word_and_ordered_script_terms() -> None: + query = analyze_script_query("OpenAI 适者生存，サバイバル") + + assert query.word_text == "OpenAI" + assert query.gram_phrases == ( + ("适者", "者生", "生存"), + ("サバ", "バイ", "イバ", "バル"), + ) + + +def test_analyze_script_query_preserves_adjoining_word_and_script_token() -> None: + query = analyze_script_query("foo適者bar") + + assert query.word_text is None + assert query.gram_phrases[0] == ("適者",) + assert ("bmprefixbefore1x0x666f6f",) in query.gram_phrases + assert ("bmprefixafter1x0x626172",) in query.gram_phrases + assert any(phrase[0].startswith("bmrole") for phrase in query.gram_phrases) + + +def test_analyze_script_query_preserves_punctuation_separated_mixed_token() -> None: + query = analyze_script_query("foo-適者-bar") + + assert query.word_text is None + assert query.gram_phrases[0] == ("適者",) + assert ("bmprefixbefore1x0x666f6f",) in query.gram_phrases + assert ("bmprefixafter1x0x626172",) in query.gram_phrases + assert any(phrase[0].startswith("bmrole") for phrase in query.gram_phrases) + + +def test_analyze_script_query_does_not_require_script_substring_in_word_channel() -> None: + query = analyze_script_query("foo適者") + + assert query.word_text is None + assert query.gram_phrases[0] == ("適者",) + assert ("bmprefixbefore1x0x666f6f",) in query.gram_phrases + assert any(phrase[0].startswith("bmrole") for phrase in query.gram_phrases) + + +def test_analyze_script_query_preserves_compatibility_bytes_in_mixed_prefix() -> None: + query = analyze_script_query("ＡＢＣ適者") + + assert query.word_text is None + assert query.gram_phrases[0] == ("適者",) + assert ("bmprefixbefore1x0x616263",) in query.gram_phrases + assert any(phrase[0].startswith("bmrole") for phrase in query.gram_phrases) + + +def test_analyze_script_query_retains_trailing_word_in_auxiliary_channel() -> None: + query = analyze_script_query("適者OpenAI") + + assert query.word_text is None + assert query.gram_phrases[0] == ("適者",) + assert ("bmprefixafter1x0x6f70656e6169",) in query.gram_phrases + assert any(phrase[0].startswith("bmrole") for phrase in query.gram_phrases) + + +def test_analyze_script_query_preserves_explicit_boolean_semantics() -> None: + query = analyze_script_query("OpenAI OR 适者生存") + + assert query.word_text == "OpenAI OR 适者生存" + assert query.gram_phrases == () + + +@pytest.mark.parametrize("text", ["NOT 生存", "生存 OR"]) +def test_analyze_script_query_preserves_boundary_boolean_operators(text: str) -> None: + query = analyze_script_query(text) + + assert query.word_text == text + assert query.gram_phrases == () + + +def test_analyze_script_query_treats_lowercase_boolean_words_as_natural_language() -> None: + query = analyze_script_query("OpenAI and 适者生存") + + assert query.word_text == "OpenAI and" + assert query.gram_phrases == (("适者", "者生", "生存"),) + + +def test_analyze_script_query_preserves_quoted_mixed_script_semantics() -> None: + query = analyze_script_query('"OpenAI 适者生存"') + + assert query.word_text == '"OpenAI 适者生存"' + assert query.gram_phrases == () + + +def test_analyze_script_query_preserves_compatibility_characters_in_word_text() -> None: + query = analyze_script_query("ＡＢＣ finance 适者生存") + + assert query.word_text == "ＡＢＣ finance" + assert query.gram_phrases == (("适者", "者生", "生存"),) + + +def test_analyze_script_query_treats_compatibility_boolean_text_as_natural_language() -> None: + query = analyze_script_query("适者 ＡＮＤ 生存") + + assert query.word_text == "ＡＮＤ" + assert query.gram_phrases == (("适者",), ("生存",)) + + +@pytest.mark.parametrize("separator", ["\t", "\n"]) +def test_analyze_script_query_matches_backend_boolean_whitespace(separator: str) -> None: + query = analyze_script_query(f"适者{separator}AND{separator}生存") + + assert query.word_text == "and" + assert query.gram_phrases == (("适者",), ("生存",)) + + +@pytest.mark.parametrize("text", ["!!!", "😀"]) +def test_analyze_script_query_preserves_punctuation_only_text(text: str) -> None: + query = analyze_script_query(text) + + assert query.word_text == text + assert query.gram_phrases == () + + +@pytest.mark.asyncio +async def test_compatibility_boolean_text_uses_script_substring_search(search_repository) -> None: + now = datetime.now(timezone.utc) + row = SearchIndexRow( + project_id=search_repository.project_id, + id=1307, + type="entity", + file_path="notes/compatibility-boolean.md", + title="Compatibility Boolean", + content_stems="不适者 ＡＮＤ 生存者", + content_snippet="不适者 ＡＮＤ 生存者", + permalink="notes/compatibility-boolean", + created_at=now, + updated_at=now, + ) + await search_repository.index_item(row) + + results = await search_repository.search("适者 ＡＮＤ 生存") + + assert [result.id for result in results] == [1307] + + +@pytest.mark.asyncio +async def test_non_space_boolean_text_uses_script_substring_search(search_repository) -> None: + now = datetime.now(timezone.utc) + row = SearchIndexRow( + project_id=search_repository.project_id, + id=1308, + type="entity", + file_path="notes/non-space-boolean.md", + title="Non-space Boolean", + content_stems="不适者\tAND\t生存者", + content_snippet="不适者\tAND\t生存者", + permalink="notes/non-space-boolean", + created_at=now, + updated_at=now, + ) + await search_repository.index_item(row) + + results = await search_repository.search("适者\tAND\t生存") + + assert [result.id for result in results] == [1308] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("query", ["!!!", "😀"]) +async def test_punctuation_only_search_does_not_return_every_row( + search_repository, + query: str, +) -> None: + now = datetime.now(timezone.utc) + row = SearchIndexRow( + project_id=search_repository.project_id, + id=1310, + type="entity", + file_path="notes/punctuation-decoy.md", + title="Punctuation decoy", + content_stems="ordinary searchable words", + content_snippet="ordinary searchable words", + permalink="notes/punctuation-decoy", + created_at=now, + updated_at=now, + ) + await search_repository.index_item(row) + + assert await search_repository.search(query) == [] + + +@pytest.mark.asyncio +async def test_sqlite_script_search_combines_metadata_and_title_filters( + search_repository, + session_maker, +) -> None: + if not isinstance(search_repository, SQLiteSearchRepository): + pytest.skip("SQLite-specific FTS5 rowid regression") + + now = datetime.now(timezone.utc) + async with db.scoped_session(session_maker) as session: + entity = Entity( + project_id=search_repository.project_id, + title="Evolution match", + note_type="note", + permalink="notes/evolution-filtered", + file_path="notes/evolution-filtered.md", + content_type="text/markdown", + entity_metadata={"region": "asia"}, + created_at=now, + updated_at=now, + ) + session.add(entity) + await session.flush() + entity_id = entity.id + + row = SearchIndexRow( + project_id=search_repository.project_id, + id=entity_id, + type="entity", + entity_id=entity_id, + file_path="notes/evolution-filtered.md", + title="Evolution match", + content_stems="不适者生存者", + content_snippet="不适者生存者", + permalink="notes/evolution-filtered", + created_at=now, + updated_at=now, + ) + await search_repository.index_item(row) + + results = await search_repository.search( + "适者", + title="Evolution match", + metadata_filters={"region": "asia"}, + ) + + assert [result.id for result in results] == [entity_id] + + +@pytest.mark.asyncio +async def test_search_matches_cjk_substring_without_matching_reordered_characters( + search_repository, +) -> None: + now = datetime.now(timezone.utc) + row = SearchIndexRow( + project_id=search_repository.project_id, + id=1294, + type="entity", + file_path="notes/evolution.md", + title="進化について", + content_stems="OpenAI and 即适者生存的讨论与黑猫，時々更新", + content_snippet="OpenAI and 即适者生存的讨论与黑猫，時々更新", + permalink="notes/evolution", + created_at=now, + updated_at=now, + ) + await search_repository.index_item(row) + + assert [result.id for result in await search_repository.search("适者生存")] == [1294] + assert [result.id for result in await search_repository.search("OpenAI 适者生存")] == [1294] + assert [result.id for result in await search_repository.search("OpenAI and 适者生存")] == [1294] + assert [result.id for result in await search_repository.search("猫")] == [1294] + assert [result.id for result in await search_repository.search("時々")] == [1294] + assert await search_repository.search("适生者存") == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("join_control", ["\u200c", "\u200d"]) +async def test_search_preserves_order_across_join_controls( + search_repository, + join_control: str, +) -> None: + now = datetime.now(timezone.utc) + rows = [ + SearchIndexRow( + project_id=search_repository.project_id, + id=1317, + type="entity", + file_path="notes/join-control-match.md", + title="Join control match", + content_stems=f"ក{join_control}ខ", + content_snippet=f"ក{join_control}ខ", + permalink="notes/join-control-match", + created_at=now, + updated_at=now, + ), + SearchIndexRow( + project_id=search_repository.project_id, + id=1318, + type="entity", + file_path="notes/join-control-reversed.md", + title="Join control reversed", + content_stems=f"ខ{join_control}ក", + content_snippet=f"ខ{join_control}ក", + permalink="notes/join-control-reversed", + created_at=now, + updated_at=now, + ), + ] + await search_repository.bulk_index_items(rows) + + results = await search_repository.search(f"ក{join_control}ខ") + + assert [result.id for result in results] == [1317] + + +@pytest.mark.asyncio +async def test_search_matches_katakana_extended_b_substring(search_repository) -> None: + now = datetime.now(timezone.utc) + row = SearchIndexRow( + project_id=search_repository.project_id, + id=1319, + type="entity", + file_path="notes/katakana-extended-b.md", + title="Katakana extended B", + content_stems="\U0001aff0\U0001aff3\U0001affd", + content_snippet="\U0001aff0\U0001aff3\U0001affd", + permalink="notes/katakana-extended-b", + created_at=now, + updated_at=now, + ) + await search_repository.index_item(row) + + results = await search_repository.search("\U0001aff3") + + assert [result.id for result in results] == [1319] + + +@pytest.mark.asyncio +async def test_search_preserves_adjoining_word_and_script_token(search_repository) -> None: + now = datetime.now(timezone.utc) + row = SearchIndexRow( + project_id=search_repository.project_id, + id=1311, + type="entity", + file_path="notes/adjoining-script.md", + title="Adjoining script", + content_stems="foo適者bar", + content_snippet="foo適者bar", + permalink="notes/adjoining-script", + created_at=now, + updated_at=now, + ) + await search_repository.index_item(row) + + results = await search_repository.search("foo適者bar") + + assert [result.id for result in results] == [1311] + + +@pytest.mark.asyncio +async def test_search_matches_script_substring_inside_longer_mixed_token(search_repository) -> None: + now = datetime.now(timezone.utc) + row = SearchIndexRow( + project_id=search_repository.project_id, + id=1312, + type="entity", + file_path="notes/longer-adjoining-script.md", + title="Longer adjoining script", + content_stems="foo不適者bar", + content_snippet="foo不適者bar", + permalink="notes/longer-adjoining-script", + created_at=now, + updated_at=now, + ) + await search_repository.index_item(row) + + results = await search_repository.search("foo適者") + + assert [result.id for result in results] == [1312] + + +@pytest.mark.asyncio +async def test_search_preserves_prefix_matching_in_mixed_token(search_repository) -> None: + now = datetime.now(timezone.utc) + row = SearchIndexRow( + project_id=search_repository.project_id, + id=1320, + type="entity", + file_path="notes/mixed-prefix.md", + title="Mixed prefix", + content_stems="foobar不適者", + content_snippet="foobar不適者", + permalink="notes/mixed-prefix", + created_at=now, + updated_at=now, + ) + await search_repository.index_item(row) + + results = await search_repository.search("foo適者") + + assert [result.id for result in results] == [1320] + + +@pytest.mark.asyncio +async def test_search_preserves_long_prefix_matching_in_mixed_token(search_repository) -> None: + now = datetime.now(timezone.utc) + indexed_prefix = "a" * 70 + query_prefix = "a" * 65 + row = SearchIndexRow( + project_id=search_repository.project_id, + id=1325, + type="entity", + file_path="notes/long-mixed-prefix.md", + title="Long mixed prefix", + content_stems=f"{indexed_prefix}適者", + content_snippet=f"{indexed_prefix}適者", + permalink="notes/long-mixed-prefix", + created_at=now, + updated_at=now, + ) + await search_repository.index_item(row) + + results = await search_repository.search(f"{query_prefix}適者") + + assert [result.id for result in results] == [1325] + + +@pytest.mark.asyncio +async def test_search_preserves_word_fragment_order_in_mixed_token(search_repository) -> None: + now = datetime.now(timezone.utc) + rows = [ + SearchIndexRow( + project_id=search_repository.project_id, + id=1321, + type="entity", + file_path="notes/mixed-order-match.md", + title="Mixed order match", + content_stems="foo適者bar", + content_snippet="foo適者bar", + permalink="notes/mixed-order-match", + created_at=now, + updated_at=now, + ), + SearchIndexRow( + project_id=search_repository.project_id, + id=1322, + type="entity", + file_path="notes/mixed-order-reversed.md", + title="Mixed order reversed", + content_stems="bar適者foo", + content_snippet="bar適者foo", + permalink="notes/mixed-order-reversed", + created_at=now, + updated_at=now, + ), + ] + await search_repository.bulk_index_items(rows) + + results = await search_repository.search("foo適者bar") + + assert [result.id for result in results] == [1321] + + +@pytest.mark.asyncio +async def test_search_binds_word_positions_to_their_script_run(search_repository) -> None: + now = datetime.now(timezone.utc) + rows = [ + SearchIndexRow( + project_id=search_repository.project_id, + id=1326, + type="entity", + file_path="notes/script-role-match.md", + title="Script role match", + content_stems="foo適者bar", + content_snippet="foo適者bar", + permalink="notes/script-role-match", + created_at=now, + updated_at=now, + ), + SearchIndexRow( + project_id=search_repository.project_id, + id=1327, + type="entity", + file_path="notes/script-role-decoy.md", + title="Script role decoy", + content_stems="foo生存bar 適者", + content_snippet="foo生存bar 適者", + permalink="notes/script-role-decoy", + created_at=now, + updated_at=now, + ), + ] + await search_repository.bulk_index_items(rows) + + results = await search_repository.search("foo適者bar") + + assert [result.id for result in results] == [1326] + + +@pytest.mark.asyncio +async def test_search_preserves_same_side_word_fragment_order(search_repository) -> None: + now = datetime.now(timezone.utc) + rows = [ + SearchIndexRow( + project_id=search_repository.project_id, + id=1323, + type="entity", + file_path="notes/same-side-order-match.md", + title="Same-side order match", + content_stems="foo-bar-baz適者", + content_snippet="foo-bar-baz適者", + permalink="notes/same-side-order-match", + created_at=now, + updated_at=now, + ), + SearchIndexRow( + project_id=search_repository.project_id, + id=1324, + type="entity", + file_path="notes/same-side-order-reversed.md", + title="Same-side order reversed", + content_stems="bar-foo-baz適者", + content_snippet="bar-foo-baz適者", + permalink="notes/same-side-order-reversed", + created_at=now, + updated_at=now, + ), + ] + await search_repository.bulk_index_items(rows) + + results = await search_repository.search("foo-bar-baz適者") + + assert [result.id for result in results] == [1323] + + +@pytest.mark.asyncio +async def test_search_preserves_compatibility_bytes_in_mixed_prefix(search_repository) -> None: + now = datetime.now(timezone.utc) + row = SearchIndexRow( + project_id=search_repository.project_id, + id=1314, + type="entity", + file_path="notes/compatibility-prefix-script.md", + title="Compatibility prefix script", + content_stems="ＡＢＣ適者", + content_snippet="ＡＢＣ適者", + permalink="notes/compatibility-prefix-script", + created_at=now, + updated_at=now, + ) + await search_repository.index_item(row) + + results = await search_repository.search("ＡＢＣ適者") + + assert [result.id for result in results] == [1314] + + +@pytest.mark.asyncio +async def test_search_requires_trailing_word_in_mixed_token(search_repository) -> None: + now = datetime.now(timezone.utc) + matching_row = SearchIndexRow( + project_id=search_repository.project_id, + id=1315, + type="entity", + file_path="notes/trailing-mixed-word.md", + title="Trailing mixed word", + content_stems="適者OpenAI", + content_snippet="適者OpenAI", + permalink="notes/trailing-mixed-word", + created_at=now, + updated_at=now, + ) + nonmatching_row = SearchIndexRow( + project_id=search_repository.project_id, + id=1316, + type="entity", + file_path="notes/script-only.md", + title="Script only", + content_stems="適者", + content_snippet="適者", + permalink="notes/script-only", + created_at=now, + updated_at=now, + ) + await search_repository.index_item(matching_row) + await search_repository.index_item(nonmatching_row) + + results = await search_repository.search("適者OpenAI") + + assert [result.id for result in results] == [1315] + + +@pytest.mark.asyncio +async def test_search_preserves_punctuation_separated_mixed_token(search_repository) -> None: + now = datetime.now(timezone.utc) + row = SearchIndexRow( + project_id=search_repository.project_id, + id=1313, + type="entity", + file_path="notes/punctuation-separated-script.md", + title="Punctuation-separated script", + content_stems="foo-適者-bar", + content_snippet="foo-適者-bar", + permalink="notes/punctuation-separated-script", + created_at=now, + updated_at=now, + ) + await search_repository.index_item(row) + + results = await search_repository.search("foo-適者-bar") + + assert [result.id for result in results] == [1313] + + +@pytest.mark.asyncio +async def test_mixed_word_and_script_search_preserves_fts_ranking(search_repository) -> None: + now = datetime.now(timezone.utc) + rows = [ + SearchIndexRow( + project_id=search_repository.project_id, + id=1296, + type="entity", + file_path="notes/strong-match.md", + title="OpenAI OpenAI OpenAI", + content_stems="OpenAI 即适者生存", + content_snippet="OpenAI 即适者生存", + permalink="notes/strong-match", + created_at=now, + updated_at=now, + ), + SearchIndexRow( + project_id=search_repository.project_id, + id=1297, + type="entity", + file_path="notes/weaker-match.md", + title="Weaker match", + content_stems="OpenAI 即适者生存", + content_snippet="OpenAI 即适者生存", + permalink="notes/weaker-match", + created_at=now, + updated_at=now, + ), + ] + await search_repository.bulk_index_items(rows) + + results = await search_repository.search("OpenAI 适者生存") + + assert [result.id for result in results] == [1296, 1297] + assert all(result.score != 0.0 for result in results) + + +@pytest.mark.asyncio +async def test_sqlite_mixed_search_requires_all_words_in_one_column(search_repository) -> None: + if not isinstance(search_repository, SQLiteSearchRepository): + pytest.skip("SQLite preserves its established per-column word matching") + + now = datetime.now(timezone.utc) + rows = [ + SearchIndexRow( + project_id=search_repository.project_id, + id=1305, + type="entity", + file_path="notes/same-column.md", + title="Same column", + content_stems="alpha beta 適者生存", + content_snippet="alpha beta 適者生存", + permalink="notes/same-column", + created_at=now, + updated_at=now, + ), + SearchIndexRow( + project_id=search_repository.project_id, + id=1306, + type="entity", + file_path="notes/split-columns.md", + title="alpha", + content_stems="beta 適者生存", + content_snippet="beta 適者生存", + permalink="notes/split-columns", + created_at=now, + updated_at=now, + ), + ] + await search_repository.bulk_index_items(rows) + + results = await search_repository.search("alpha beta 適者生存") + + assert [result.id for result in results] == [1305] + + +@pytest.mark.asyncio +async def test_sqlite_relaxed_search_keeps_word_and_script_channels_distinct( + search_repository, +) -> None: + if not isinstance(search_repository, SQLiteSearchRepository): + pytest.skip("SQLite-specific relaxed FTS5 regression") + + now = datetime.now(timezone.utc) + row = SearchIndexRow( + project_id=search_repository.project_id, + id=1312, + type="entity", + file_path="notes/script-only.md", + title="Script only", + content_stems="適者", + content_snippet="適者", + permalink="notes/script-only", + created_at=now, + updated_at=now, + ) + await search_repository.index_item(row) + + results = await search_repository.search("missing 適者", allow_relaxed=True) + total = await search_repository.count("missing 適者", allow_relaxed=True) + + assert results == [] + assert total == 0 + + +@pytest.mark.asyncio +async def test_search_ranking_includes_every_script_run(search_repository) -> None: + now = datetime.now(timezone.utc) + row = SearchIndexRow( + project_id=search_repository.project_id, + id=1300, + type="entity", + file_path="notes/multiple-runs.md", + title="Multiple runs", + content_stems="适者 生存 生存", + content_snippet="适者 生存 生存", + permalink="notes/multiple-runs", + created_at=now, + updated_at=now, + ) + await search_repository.index_item(row) + + first_run_results = await search_repository.search("适者") + all_run_results = await search_repository.search("适者 生存") + + assert [result.id for result in all_run_results] == [1300] + assert all_run_results[0].score != first_run_results[0].score + + +@pytest.mark.asyncio +async def test_postgres_ranking_adds_contributions_from_every_script_run( + search_repository, +) -> None: + if not isinstance(search_repository, PostgresSearchRepository): + pytest.skip("PostgreSQL combines independently ranked script phrases") + + now = datetime.now(timezone.utc) + shared_first_run = "適者 " * 5 + rows = [ + SearchIndexRow( + project_id=search_repository.project_id, + id=1302, + type="entity", + file_path="notes/one-secondary-match.md", + title="One secondary match", + content_stems=f"{shared_first_run}生存", + content_snippet=f"{shared_first_run}生存", + permalink="notes/one-secondary-match", + created_at=now, + updated_at=now, + ), + SearchIndexRow( + project_id=search_repository.project_id, + id=1303, + type="entity", + file_path="notes/many-secondary-matches.md", + title="Many secondary matches", + content_stems=f"{shared_first_run}{'生存 ' * 5}", + content_snippet=f"{shared_first_run}{'生存 ' * 5}", + permalink="notes/many-secondary-matches", + created_at=now, + updated_at=now, + ), + ] + await search_repository.bulk_index_items(rows) + + results = await search_repository.search("適者 生存") + + assert [result.id for result in results] == [1303, 1302] + assert results[0].score > results[1].score + + +@pytest.mark.asyncio +async def test_postgres_mixed_search_ignores_empty_stopword_query(search_repository) -> None: + if not isinstance(search_repository, PostgresSearchRepository): + pytest.skip("PostgreSQL's English dictionary removes stopwords") + + now = datetime.now(timezone.utc) + row = SearchIndexRow( + project_id=search_repository.project_id, + id=1304, + type="entity", + file_path="notes/stopword-and-script.md", + title="Script match", + content_stems="適者生存", + content_snippet="適者生存", + permalink="notes/stopword-and-script", + created_at=now, + updated_at=now, + ) + await search_repository.index_item(row) + + results = await search_repository.search("the 適者生存") + + assert [result.id for result in results] == [1304] + + +@pytest.mark.asyncio +async def test_quoted_mixed_script_search_preserves_phrase_adjacency(search_repository) -> None: + if not isinstance(search_repository, SQLiteSearchRepository): + pytest.skip("SQLite-specific quoted FTS5 regression") + + now = datetime.now(timezone.utc) + rows = [ + SearchIndexRow( + project_id=search_repository.project_id, + id=1298, + type="entity", + file_path="notes/adjacent.md", + title="Adjacent", + content_stems="OpenAI 适者生存", + content_snippet="OpenAI 适者生存", + permalink="notes/adjacent", + created_at=now, + updated_at=now, + ), + SearchIndexRow( + project_id=search_repository.project_id, + id=1299, + type="entity", + file_path="notes/separated.md", + title="Separated", + content_stems="OpenAI words far away from 适者生存", + content_snippet="OpenAI words far away from 适者生存", + permalink="notes/separated", + created_at=now, + updated_at=now, + ), + ] + await search_repository.bulk_index_items(rows) + + results = await search_repository.search('"OpenAI 适者生存"') + + assert [result.id for result in results] == [1298] + + +@pytest.mark.asyncio +async def test_word_search_preserves_nfkc_sensitive_compatibility_characters( + search_repository, +) -> None: + now = datetime.now(timezone.utc) + row = SearchIndexRow( + project_id=search_repository.project_id, + id=1301, + type="entity", + file_path="notes/compatibility.md", + title="Compatibility", + content_stems="ＡＢＣ finance", + content_snippet="ＡＢＣ finance", + permalink="notes/compatibility", + created_at=now, + updated_at=now, + ) + await search_repository.index_item(row) + + results = await search_repository.search("ＡＢＣ finance") + + assert [result.id for result in results] == [1301] + + +@pytest.mark.asyncio +async def test_search_matches_script_text_beyond_the_parent_fts_limit(search_repository) -> None: + now = datetime.now(timezone.utc) + row = SearchIndexRow( + project_id=search_repository.project_id, + id=1295, + type="entity", + file_path="notes/long.md", + title="進化 Long note", + content_stems="bounded parent search text", + content_snippet=f"{'x' * 9_000} 适者生存", + permalink="notes/long", + created_at=now, + updated_at=now, + ) + await search_repository.index_item(row) + + assert [result.id for result in await search_repository.search("适者生存")] == [1295] + assert [result.id for result in await search_repository.search("進化 适者生存")] == [1295] diff --git a/tests/repository/test_search_repository.py b/tests/repository/test_search_repository.py index 8eb049033..ee9d40c8b 100644 --- a/tests/repository/test_search_repository.py +++ b/tests/repository/test_search_repository.py @@ -282,6 +282,32 @@ async def test_sqlite_text_search_matches_full_content_snippet(search_repository assert await search_repository.count(search_text=marker) == 1 +@pytest.mark.asyncio +async def test_sqlite_word_query_keeps_terms_in_one_search_column(search_repository, search_entity): + """Adding the script channel must not broaden established word-query matches.""" + if is_postgres_backend(search_repository): + pytest.skip("SQLite's per-column FTS matching is backend-specific") + + search_row = SearchIndexRow( + id=search_entity.id, + type=SearchItemType.ENTITY.value, + title="alpha", + content_stems="beta", + content_snippet="beta", + permalink=search_entity.permalink, + file_path=search_entity.file_path, + entity_id=search_entity.id, + metadata={"note_type": search_entity.note_type}, + created_at=search_entity.created_at, + updated_at=search_entity.updated_at, + project_id=search_repository.project_id, + ) + await search_repository.index_item(search_row) + + assert await search_repository.search(search_text="alpha beta") == [] + assert await search_repository.count(search_text="alpha beta") == 0 + + @pytest.mark.asyncio async def test_index_item_upsert_on_duplicate_permalink(search_repository, search_entity): """Test that indexing the same permalink twice uses upsert instead of failing. @@ -1265,10 +1291,10 @@ async def test_multiword_query_relaxes_to_or_when_strict_misses(search_repositor @pytest.mark.asyncio -async def test_cjk_compound_query_relaxes_with_backend_prefix_terms( +async def test_cjk_compound_query_matches_with_or_without_relaxation( search_repository, search_entity ): - """Whitespace-separated CJK terms should match indexed CJK compounds when relaxed.""" + """Script n-grams make whitespace-separated CJK terms strict matches.""" row = SearchIndexRow( project_id=search_repository.project_id, id=search_entity.id, @@ -1286,7 +1312,7 @@ async def test_cjk_compound_query_relaxes_with_backend_prefix_terms( await search_repository.index_item(row) strict = await search_repository.search(search_text="季度 报告") - assert strict == [] + assert any(r.entity_id == search_entity.id for r in strict) results = await search_repository.search(search_text="季度 报告", allow_relaxed=True) assert any(r.entity_id == search_entity.id for r in results) diff --git a/tests/repository/test_search_text_with_metadata_filters.py b/tests/repository/test_search_text_with_metadata_filters.py index 7ac878ab3..6eef72b10 100644 --- a/tests/repository/test_search_text_with_metadata_filters.py +++ b/tests/repository/test_search_text_with_metadata_filters.py @@ -7,6 +7,7 @@ from basic_memory import db from basic_memory.models.knowledge import Entity from basic_memory.repository.search_index_row import SearchIndexRow +from basic_memory.repository.sqlite_search_repository import SQLiteSearchRepository from basic_memory.schemas.search import SearchItemType @@ -63,3 +64,34 @@ async def test_search_text_and_metadata_filters_work_together(search_repository, ) assert {row.id for row in results} == {active.id} + + +@pytest.mark.asyncio +async def test_sqlite_script_search_with_metadata_filters_preserves_ranking( + search_repository, + session_maker, +) -> None: + if not isinstance(search_repository, SQLiteSearchRepository): + pytest.skip("SQLite-specific FTS5 ranking regression") + + stronger = await _index_entity( + search_repository, + session_maker, + "適者生存 適者生存 適者生存", + "active", + ) + weaker = await _index_entity( + search_repository, + session_maker, + "適者生存", + "active", + ) + + results = await search_repository.search( + search_text="適者生存", + metadata_filters={"status": "active"}, + ) + + assert [row.id for row in results] == [stronger.id, weaker.id] + assert results[0].score != results[1].score + assert all(row.score != 0.0 for row in results) diff --git a/tests/test_script_ngram_search_migration.py b/tests/test_script_ngram_search_migration.py new file mode 100644 index 000000000..7efe08ed8 --- /dev/null +++ b/tests/test_script_ngram_search_migration.py @@ -0,0 +1,103 @@ +"""Migration coverage for the portable script n-gram FTS channel.""" + +import sqlite3 +from importlib import import_module +from types import SimpleNamespace + +from alembic.migration import MigrationContext +from alembic.operations import Operations +from sqlalchemy import create_engine + + +migration = import_module( + "basic_memory.alembic.versions.d2e3f4a5b6c7_add_script_ngrams_to_full_text_search" +) + + +def test_sqlite_upgrade_and_downgrade_preserve_word_search_rows(tmp_path, monkeypatch) -> None: + database_path = tmp_path / "script-ngrams.db" + engine = create_engine(f"sqlite:///{database_path}") + with engine.begin() as connection: + connection.exec_driver_sql(""" + CREATE VIRTUAL TABLE search_index USING fts5( + id UNINDEXED, title, content_stems, content_snippet, permalink, + file_path UNINDEXED, type UNINDEXED, project_id UNINDEXED, + from_id UNINDEXED, to_id UNINDEXED, relation_type UNINDEXED, + entity_id UNINDEXED, category UNINDEXED, metadata UNINDEXED, + created_at UNINDEXED, updated_at UNINDEXED, + tokenize='unicode61 tokenchars 0x2F', prefix='1,2,3,4' + ) + """) + connection.exec_driver_sql(""" + INSERT INTO search_index (id, title, content_stems, project_id) + VALUES (1, 'Existing title', 'existing words', 7) + """) + monkeypatch.setattr( + migration, + "op", + Operations(MigrationContext.configure(connection)), + ) + migration.upgrade() + + with sqlite3.connect(database_path) as connection: + columns = [row[1] for row in connection.execute("PRAGMA table_info(search_index)")] + assert "script_ngrams" in columns + assert connection.execute( + "SELECT id, title, script_ngrams FROM search_index" + ).fetchall() == [(1, "Existing title", "")] + + with engine.begin() as connection: + monkeypatch.setattr( + migration, + "op", + Operations(MigrationContext.configure(connection)), + ) + migration.downgrade() + + with sqlite3.connect(database_path) as connection: + columns = [row[1] for row in connection.execute("PRAGMA table_info(search_index)")] + assert "script_ngrams" not in columns + assert connection.execute("SELECT id, title FROM search_index").fetchall() == [ + (1, "Existing title") + ] + + +def test_sqlite_upgrade_leaves_runtime_search_index_creation_to_the_model( + tmp_path, monkeypatch +) -> None: + database_path = tmp_path / "fresh-script-ngrams.db" + engine = create_engine(f"sqlite:///{database_path}") + with engine.begin() as connection: + monkeypatch.setattr( + migration, + "op", + Operations(MigrationContext.configure(connection)), + ) + migration.upgrade() + + with sqlite3.connect(database_path) as connection: + search_index_exists = connection.execute( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'search_index'" + ).fetchone() + assert search_index_exists is None + + +def test_postgres_upgrade_and_downgrade_manage_both_script_indexes(monkeypatch) -> None: + statements: list[str] = [] + monkeypatch.setattr( + migration.op, + "get_bind", + lambda: SimpleNamespace(dialect=SimpleNamespace(name="postgresql")), + ) + monkeypatch.setattr( + migration.op, "execute", lambda statement: statements.append(str(statement)) + ) + + migration.upgrade() + migration.downgrade() + + sql = "\n".join(statements) + assert "idx_search_index_script_ngrams_fts" in sql + assert "idx_search_index_fts_chunks_script_ngrams_fts" in sql + assert "to_tsvector('simple', script_ngrams)" in sql + assert "DROP COLUMN IF EXISTS script_ngrams" in sql