feat(fts): add ngram tokenizer

python/zvec/model/param/__init__.pyi

@@ -834,8 +834,8 @@ class FtsIndexParam(IndexParam):
 
     Attributes:
         type (IndexType): Always ``IndexType.FTS``.
-        tokenizer_name (str): Name of the tokenizer (one of "standard", "jieba",
-            "whitespace").
+        tokenizer_name (str): Name of the tokenizer (one of "standard", "ngram",
+            "jieba", "whitespace").
             Default is "standard".
         filters (list[str]): List of token filter names applied after tokenization.
             Supported filters are "lowercase", "ascii_folding", and "stemmer".
@@ -845,6 +845,13 @@ class FtsIndexParam(IndexParam):
             Tokenizers:
                 standard:
                     - "max_token_length" (positive integer).
+                ngram:
+                    - "ngram_min" (positive integer, default 2).
+                    - "ngram_max" (positive integer, default 2).
+                    - "token_chars" (array of "letter", "digit",
+                      "whitespace", "punctuation", "symbol"; default [] keeps
+                      all valid UTF-8 characters). custom_token_chars is not
+                      supported.
                 jieba:
                     - "jieba_dict_dir" (directory containing jieba.dict.utf8 and
                       hmm_model.utf8).
@@ -892,6 +899,13 @@ class FtsIndexParam(IndexParam):
                 Tokenizers:
                     standard:
                         - "max_token_length" (positive integer).
+                    ngram:
+                        - "ngram_min" (positive integer, default 2).
+                        - "ngram_max" (positive integer, default 2).
+                        - "token_chars" (array of "letter", "digit",
+                          "whitespace", "punctuation", "symbol"; default []
+                          keeps all valid UTF-8 characters). custom_token_chars
+                          is not supported.
                     jieba:
                         - "jieba_dict_dir".
                         - "user_dict_path".

src/binding/python/model/param/python_param.cc

@@ -257,8 +257,8 @@ Controls the tokenizer pipeline used during indexing and querying.
 
 Attributes:
     type (IndexType): Always ``IndexType.FTS``.
-    tokenizer_name (str): Name of the tokenizer (one of "standard", "jieba",
-        "whitespace").
+    tokenizer_name (str): Name of the tokenizer (one of "standard", "ngram",
+        "jieba", "whitespace").
         Default is "standard".
     filters (list[str]): List of token filter names applied after tokenization.
         Supported values include "lowercase", "ascii_folding", and "stemmer".
@@ -268,6 +268,12 @@ Controls the tokenizer pipeline used during indexing and querying.
         Tokenizers:
             standard:
                 - "max_token_length" (positive integer).
+            ngram:
+                - "ngram_min" (positive integer, default 2).
+                - "ngram_max" (positive integer, default 2).
+                - "token_chars" (array of "letter", "digit", "whitespace",
+                  "punctuation", "symbol"; default [] keeps all valid UTF-8
+                  characters). custom_token_chars is not supported.
             jieba:
                 - "jieba_dict_dir" (directory containing jieba.dict.utf8 and
                   hmm_model.utf8).
@@ -316,6 +322,12 @@ Constructs an FtsIndexParam instance.
         Tokenizers:
             standard:
                 - "max_token_length" (positive integer).
+            ngram:
+                - "ngram_min" (positive integer, default 2).
+                - "ngram_max" (positive integer, default 2).
+                - "token_chars" (array of "letter", "digit", "whitespace",
+                  "punctuation", "symbol"; default [] keeps all valid UTF-8
+                  characters). custom_token_chars is not supported.
             jieba:
                 - "jieba_dict_dir".
                 - "user_dict_path".

src/db/index/column/fts_column/fts_types.h

@@ -51,7 +51,7 @@ struct FtsSegmentStats {
 };
 
 struct FtsIndexParams {
-  // Supported tokenizers: standard, jieba, whitespace.
+  // Supported tokenizers: standard, ngram, jieba, whitespace.
   std::string tokenizer_name{"standard"};
   // Supported filters: lowercase, ascii_folding.
   std::vector<std::string> filters{"lowercase"};

src/db/index/column/fts_column/tokenizer/jieba_tokenizer.cc

@@ -44,7 +44,7 @@ static std::string resolve_jieba_dict_dir(const ailego::JsonObject &config) {
   return GlobalConfig::Instance().jieba_dict_dir();
 }
 
-bool JiebaTokenizer::init(const ailego::JsonObject &config) {
+Status JiebaTokenizer::init(const ailego::JsonObject &config) {
   std::string user_dict_path =
       get_string_or_default(config, "user_dict_path", "");
 
@@ -58,21 +58,20 @@ bool JiebaTokenizer::init(const ailego::JsonObject &config) {
   } else if (mode_str == "hmm") {
     cut_mode_ = CutMode::kHmm;
   } else {
-    LOG_ERROR("JiebaTokenizer: unknown cut_mode '%s'", mode_str.c_str());
-    return false;
+    return Status::InvalidArgument("JiebaTokenizer: unknown cut_mode '",
+                                   mode_str, "'");
   }
 
   bool needs_dict = cut_mode_ != CutMode::kHmm;
   bool needs_model = cut_mode_ != CutMode::kFull;
 
   std::string dict_dir = resolve_jieba_dict_dir(config);
   if ((needs_dict || needs_model) && dict_dir.empty()) {
-    LOG_ERROR(
+    return Status::InvalidArgument(
         "JiebaTokenizer: jieba_dict_dir not configured. Set via "
         "extra_params.jieba_dict_dir, ZVEC_JIEBA_DICT_DIR env var, "
         "or zvec.set_default_jieba_dict_dir() / "
         "zvec.init(jieba_dict_dir=...).");
-    return false;
   }
 
   std::string dict_path = needs_dict ? dict_dir + "/jieba.dict.utf8" : "";
@@ -107,13 +106,13 @@ bool JiebaTokenizer::init(const ailego::JsonObject &config) {
   } catch (const std::exception &e) {
     LOG_ERROR("JiebaTokenizer init failed: %s", e.what());
     reset();
-    return false;
+    return Status::InvalidArgument("JiebaTokenizer init failed: ", e.what());
   }
 
   initialized_ = true;
   LOG_INFO("JiebaTokenizer init success. dict_dir[%s] cut_mode[%s]",
            dict_dir.c_str(), mode_str.c_str());
-  return true;
+  return Status::OK();
 }
 
 JiebaTokenizer::~JiebaTokenizer() = default;

src/db/index/column/fts_column/tokenizer/jieba_tokenizer.h

@@ -47,7 +47,7 @@ class JiebaTokenizer : public Tokenizer {
   // jieba_dict_dir resolution: per-field > ZVEC_JIEBA_DICT_DIR >
   // zvec::GlobalConfig::jieba_dict_dir() (set by SDK on import or via init).
   // Stop-word filtering belongs to a TokenFilter, not here.
-  bool init(const ailego::JsonObject &config) override;
+  Status init(const ailego::JsonObject &config) override;
 
   std::vector<Token> tokenize(const std::string &text) const override;