feat(core): support unsegmented-script full-text search

docs/semantic-search.md

@@ -203,17 +203,21 @@ use the same model contract.
 
 ### Chinese, Japanese, Korean and other unsegmented scripts
 
-Keyword (full-text) search does not segment CJK text. Both backends tokenize a run of CJK
-characters as a single token — SQLite's FTS5 `unicode61` tokenizer and Postgres's default text
-search parser alike. Query terms are matched as prefixes, so `生存` finds a note containing
-`生存竞争` (the run starts with the query) but not one containing only `适者生存`, where the
-query sits in the middle of a run.
-
-Until a segmenting tokenizer option ships (tracked in
-[#1294](https://github.com/basicmachines-co/basic-memory/issues/1294)), use semantic or hybrid
-search with a multilingual embedding model for these languages — the Jina Chinese-English model
-or multilingual E5 configured above both work — and expect keyword search to match whole runs
-and run prefixes only.
+Full-text search automatically adds an ordered script n-gram channel for writing systems that do
+not reliably separate words with spaces. It covers Han, Hiragana, Katakana, Hangul, Bopomofo,
+Thai, Lao, Tibetan, Myanmar, and Khmer text on both SQLite and Postgres. For example, `生存`
+matches the middle of `适者生存`, while a query with the characters in a different order does
+not. Mixed queries such as `OpenAI 适者生存` require both the word and script terms.
+
+There is no language, tokenizer, or embedding setting to enable. New and edited notes are indexed
+automatically. After upgrading an existing local project, run `bm reindex --full --search` once
+to force every existing note through search indexing and populate its script terms. Hosted Cloud
+projects receive the same Postgres analysis through the managed full fleet reindex; Cloud users
+do not run a local command.
+
+This is lexical matching, independent of the configured embedding model. Vector and hybrid search
+quality in these languages still depends on choosing a multilingual embedding model such as the
+Jina Chinese-English model or multilingual E5 configured above.
 
 ### OpenAI
 

src/basic_memory/alembic/versions/d2e3f4a5b6c7_add_script_ngrams_to_full_text_search.py

@@ -0,0 +1,123 @@
+"""Add portable script n-grams to full-text search.
+
+Revision ID: d2e3f4a5b6c7
+Revises: bcdbd5a942ca
+Create Date: 2026-08-29 00:00:00.000000
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+from sqlalchemy import text
+
+
+revision: str = "d2e3f4a5b6c7"
+down_revision: Union[str, None] = "bcdbd5a942ca"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+SQLITE_COLUMNS = """
+    id, title, content_stems, content_snippet, {script_column} permalink,
+    file_path, type, project_id, from_id, to_id, relation_type, entity_id,
+    category, metadata, created_at, updated_at
+"""
+
+
+def rebuild_sqlite_search_index(*, include_script_ngrams: bool) -> None:
+    """Copy the FTS5 table while changing its indexed-column contract."""
+    bind = op.get_bind()
+    search_index_sql: str | None = bind.execute(
+        text("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'search_index'")
+    ).scalar_one_or_none()
+
+    # The runtime owns creation of the derived FTS table, so migrations only transform an
+    # existing FTS5 index. This also leaves unrelated physical tables with the same name alone.
+    if search_index_sql is None or "using fts5" not in search_index_sql.casefold():
+        return
+
+    script_definition = "script_ngrams," if include_script_ngrams else ""
+    op.execute(f"""
+        CREATE VIRTUAL TABLE search_index_rebuilt USING fts5(
+            id UNINDEXED,
+            title,
+            content_stems,
+            content_snippet,
+            {script_definition}
+            permalink,
+            file_path UNINDEXED,
+            type UNINDEXED,
+            project_id UNINDEXED,
+            from_id UNINDEXED,
+            to_id UNINDEXED,
+            relation_type UNINDEXED,
+            entity_id UNINDEXED,
+            category UNINDEXED,
+            metadata UNINDEXED,
+            created_at UNINDEXED,
+            updated_at UNINDEXED,
+            tokenize='unicode61 tokenchars 0x2F',
+            prefix='1,2,3,4'
+        )
+    """)
+
+    source_columns = SQLITE_COLUMNS.format(script_column="")
+    target_columns = SQLITE_COLUMNS.format(
+        script_column="script_ngrams," if include_script_ngrams else ""
+    )
+    selected_columns = SQLITE_COLUMNS.format(script_column="'' AS script_ngrams,")
+    if not include_script_ngrams:
+        selected_columns = source_columns
+    op.execute(
+        f"INSERT INTO search_index_rebuilt ({target_columns}) "
+        f"SELECT {selected_columns} FROM search_index"
+    )
+    op.execute("DROP TABLE search_index")
+    op.execute("ALTER TABLE search_index_rebuilt RENAME TO search_index")
+
+
+def upgrade() -> None:
+    """Add the derived script channel; a reindex populates existing rows."""
+    if op.get_bind().dialect.name == "sqlite":
+        rebuild_sqlite_search_index(include_script_ngrams=True)
+        return
+
+    op.execute("ALTER TABLE search_index ADD COLUMN script_ngrams TEXT NOT NULL DEFAULT ''")
+    op.execute("""
+        ALTER TABLE search_index
+        ADD COLUMN script_ngrams_index_col tsvector GENERATED ALWAYS AS (
+            to_tsvector('simple', script_ngrams)
+        ) STORED
+    """)
+    op.execute("""
+        CREATE INDEX idx_search_index_script_ngrams_fts
+        ON search_index USING gin(script_ngrams_index_col)
+    """)
+    op.execute(
+        "ALTER TABLE search_index_fts_chunks ADD COLUMN script_ngrams TEXT NOT NULL DEFAULT ''"
+    )
+    op.execute("""
+        ALTER TABLE search_index_fts_chunks
+        ADD COLUMN script_ngrams_index_col tsvector GENERATED ALWAYS AS (
+            to_tsvector('simple', script_ngrams)
+        ) STORED
+    """)
+    op.execute("""
+        CREATE INDEX idx_search_index_fts_chunks_script_ngrams_fts
+        ON search_index_fts_chunks USING gin(script_ngrams_index_col)
+    """)
+
+
+def downgrade() -> None:
+    """Remove the script channel while preserving the existing word index."""
+    if op.get_bind().dialect.name == "sqlite":
+        rebuild_sqlite_search_index(include_script_ngrams=False)
+        return
+
+    op.execute("DROP INDEX IF EXISTS idx_search_index_fts_chunks_script_ngrams_fts")
+    op.execute("ALTER TABLE search_index_fts_chunks DROP COLUMN IF EXISTS script_ngrams_index_col")
+    op.execute("ALTER TABLE search_index_fts_chunks DROP COLUMN IF EXISTS script_ngrams")
+    op.execute("DROP INDEX IF EXISTS idx_search_index_script_ngrams_fts")
+    op.execute("ALTER TABLE search_index DROP COLUMN IF EXISTS script_ngrams_index_col")
+    op.execute("ALTER TABLE search_index DROP COLUMN IF EXISTS script_ngrams")

src/basic_memory/models/search.py

@@ -21,6 +21,7 @@
     title TEXT,
     content_stems TEXT,
     content_snippet TEXT,
+    script_ngrams TEXT NOT NULL DEFAULT '',
     permalink VARCHAR,
     file_path VARCHAR,
     type VARCHAR,
@@ -39,6 +40,9 @@
             coalesce(content_stems, '')
         )
     ) STORED,
+    script_ngrams_index_col tsvector GENERATED ALWAYS AS (
+        to_tsvector('simple', script_ngrams)
+    ) STORED,
     PRIMARY KEY (id, type, project_id),
     FOREIGN KEY (project_id) REFERENCES project(id) ON DELETE CASCADE
 )
@@ -48,6 +52,11 @@
 CREATE INDEX IF NOT EXISTS idx_search_index_fts ON search_index USING gin(textsearchable_index_col)
 """)
 
+CREATE_POSTGRES_SEARCH_INDEX_SCRIPT_NGRAMS_FTS = DDL("""
+CREATE INDEX IF NOT EXISTS idx_search_index_script_ngrams_fts
+ON search_index USING gin(script_ngrams_index_col)
+""")
+
 # Full note bodies are stored in bounded child rows so one unusually large note
 # cannot exceed PostgreSQL's per-tsvector size limit.
 CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_TABLE = DDL("""
@@ -57,9 +66,13 @@
     search_index_type VARCHAR NOT NULL,
     chunk_index INTEGER NOT NULL,
     chunk_text TEXT NOT NULL,
+    script_ngrams TEXT NOT NULL DEFAULT '',
     textsearchable_index_col tsvector GENERATED ALWAYS AS (
         to_tsvector('english', chunk_text)
     ) STORED,
+    script_ngrams_index_col tsvector GENERATED ALWAYS AS (
+        to_tsvector('simple', script_ngrams)
+    ) STORED,
     PRIMARY KEY (project_id, search_index_id, search_index_type, chunk_index),
     FOREIGN KEY (search_index_id, search_index_type, project_id)
         REFERENCES search_index(id, type, project_id)
@@ -72,6 +85,11 @@
 ON search_index_fts_chunks USING gin(textsearchable_index_col)
 """)
 
+CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_SCRIPT_NGRAMS_INDEX = DDL("""
+CREATE INDEX IF NOT EXISTS idx_search_index_fts_chunks_script_ngrams_fts
+ON search_index_fts_chunks USING gin(script_ngrams_index_col)
+""")
+
 CREATE_POSTGRES_SEARCH_INDEX_METADATA = DDL("""
 CREATE INDEX IF NOT EXISTS idx_search_index_metadata_gin ON search_index USING gin(metadata jsonb_path_ops)
 """)
@@ -94,6 +112,7 @@
     title,                 -- Title for searching
     content_stems,         -- Main searchable content split into stems
     content_snippet,       -- File content snippet for display
+    script_ngrams,         -- Portable bigrams for scripts without word boundaries
     permalink,             -- Stable identifier (now indexed for path search)
     file_path UNINDEXED,   -- Physical location
     type UNINDEXED,        -- entity/relation/observation

src/basic_memory/repository/accepted_note_search_repository.py

@@ -13,6 +13,8 @@
     ProjectIndexExternalVectorCleaner,
     delete_project_index_vector_rows,
 )
+from basic_memory.repository.postgres_fts_chunks import split_postgres_fts_chunks
+from basic_memory.repository.script_ngrams import build_script_ngrams
 
 type SearchIndexSqlValue = str | int | datetime | None
 type SearchIndexSqlParams = dict[str, SearchIndexSqlValue]
@@ -28,14 +30,15 @@
 INSERT_ACCEPTED_NOTE_SEARCH_SQL = text(
     """
     INSERT INTO search_index (
-        id, title, content_stems, content_snippet, permalink, file_path, type, metadata,
+        id, title, content_stems, content_snippet, script_ngrams,
+        permalink, file_path, type, metadata,
         from_id, to_id, relation_type,
         entity_id, category,
         created_at, updated_at,
         project_id
     ) VALUES (
-        :id, :title, :content_stems, :content_snippet, :permalink, :file_path, :type,
-        :metadata,
+        :id, :title, :content_stems, :content_snippet, :script_ngrams,
+        :permalink, :file_path, :type, :metadata,
         NULL, NULL, NULL,
         :entity_id, NULL,
         :created_at, :updated_at,
@@ -47,14 +50,15 @@
 UPSERT_ACCEPTED_NOTE_SEARCH_SQL = text(
     """
     INSERT INTO search_index (
-        id, title, content_stems, content_snippet, permalink, file_path, type, metadata,
+        id, title, content_stems, content_snippet, script_ngrams,
+        permalink, file_path, type, metadata,
         from_id, to_id, relation_type,
         entity_id, category,
         created_at, updated_at,
         project_id
     ) VALUES (
-        :id, :title, :content_stems, :content_snippet, :permalink, :file_path, :type,
-        CAST(:metadata AS jsonb),
+        :id, :title, :content_stems, :content_snippet, :script_ngrams,
+        :permalink, :file_path, :type, CAST(:metadata AS jsonb),
         NULL, NULL, NULL,
         :entity_id, NULL,
         :created_at, :updated_at,
@@ -65,6 +69,7 @@
         title = EXCLUDED.title,
         content_stems = EXCLUDED.content_stems,
         content_snippet = EXCLUDED.content_snippet,
+        script_ngrams = EXCLUDED.script_ngrams,
         file_path = EXCLUDED.file_path,
         type = EXCLUDED.type,
         metadata = EXCLUDED.metadata,
@@ -78,6 +83,42 @@
     """
 )
 
+INSERT_ACCEPTED_NOTE_FTS_CHUNKS_SQL = text(
+    """
+    INSERT INTO search_index_fts_chunks (
+        project_id,
+        search_index_id,
+        search_index_type,
+        chunk_index,
+        chunk_text,
+        script_ngrams
+    )
+    SELECT
+        :project_id,
+        chunk.search_index_id,
+        chunk.search_index_type,
+        chunk.chunk_index,
+        chunk.chunk_text,
+        chunk.script_ngrams
+    FROM jsonb_to_recordset(CAST(:chunks AS JSONB)) AS chunk(
+        search_index_id INTEGER,
+        search_index_type VARCHAR,
+        chunk_index INTEGER,
+        chunk_text TEXT,
+        script_ngrams TEXT
+    )
+    """
+)
+
+DELETE_ACCEPTED_NOTE_FTS_CHUNKS_SQL = text(
+    """
+    DELETE FROM search_index_fts_chunks
+    WHERE project_id = :project_id
+      AND search_index_id = :search_index_id
+      AND search_index_type = :search_index_type
+    """
+)
+
 
 def accepted_note_search_insert_statement(session: AsyncSession):
     """Return the insert statement supported by the active search table backend."""
@@ -88,13 +129,19 @@ def accepted_note_search_insert_statement(session: AsyncSession):
 
 def accepted_note_search_insert_params(
     row: AcceptedNoteSearchRow,
+    *,
+    include_full_content_grams: bool,
 ) -> SearchIndexSqlParams:
     """Build SQL parameters for one accepted-note search row."""
+    script_texts = (row.title, row.content_stems)
+    if include_full_content_grams:
+        script_texts = (*script_texts, row.content_snippet)
     return {
         "id": row.id,
         "title": row.title,
         "content_stems": row.content_stems,
         "content_snippet": row.content_snippet,
+        "script_ngrams": build_script_ngrams(*script_texts),
         "permalink": row.permalink,
         "file_path": row.file_path,
         "type": row.item_type,
@@ -134,10 +181,42 @@ async def refresh_entity(
             DELETE_ACCEPTED_NOTE_SEARCH_SQL,
             {"entity_id": row.entity_id, "project_id": row.project_id},
         )
+        is_sqlite = session.get_bind().dialect.name == "sqlite"
         await session.execute(
             accepted_note_search_insert_statement(session),
-            accepted_note_search_insert_params(row),
+            accepted_note_search_insert_params(
+                row,
+                include_full_content_grams=is_sqlite,
+            ),
         )
+        if is_sqlite:
+            return
+
+        # An upsert can move an older permalink owner's chunks through ON UPDATE CASCADE.
+        # Clear the final parent key before installing this accepted note's replacements.
+        await session.execute(
+            DELETE_ACCEPTED_NOTE_FTS_CHUNKS_SQL,
+            {
+                "project_id": row.project_id,
+                "search_index_id": row.id,
+                "search_index_type": row.item_type,
+            },
+        )
+        chunks = [
+            {
+                "search_index_id": row.id,
+                "search_index_type": row.item_type,
+                "chunk_index": chunk_index,
+                "chunk_text": chunk_text,
+                "script_ngrams": build_script_ngrams(chunk_text),
+            }
+            for chunk_index, chunk_text in split_postgres_fts_chunks(row.content_snippet)
+        ]
+        if chunks:
+            await session.execute(
+                INSERT_ACCEPTED_NOTE_FTS_CHUNKS_SQL,
+                {"project_id": row.project_id, "chunks": json.dumps(chunks)},
+            )
 
     async def delete_entity(
         self,