diff --git a/CMakeLists.txt b/CMakeLists.txt index 6a4d92252..206a1c5ff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1682,6 +1682,12 @@ target_link_libraries(tokenizer_test PRIVATE onebit_npu) add_executable(tokenizer_linear_test tests/tokenizer_linear_test.cpp) target_link_libraries(tokenizer_linear_test PRIVATE onebit_npu) add_test(NAME tokenizer_linear COMMAND tokenizer_linear_test ${CMAKE_SOURCE_DIR}/tests/data/tokenizer_linear.json) +# A vocabulary whose normalizer rewrites spaces to the metaspace symbol without prepending one (Gemma4's shape): +# ids held to the reference library's on a synthetic nine-token file; no model needed, runs in CI. +add_executable(tokenizer_replace_meta_test tests/tokenizer_replace_meta_test.cpp) +target_link_libraries(tokenizer_replace_meta_test PRIVATE onebit_npu) +add_test(NAME tokenizer_replace_meta COMMAND tokenizer_replace_meta_test ${CMAKE_SOURCE_DIR}/tests/data/tokenizer_replace_meta.json) +add_dependencies(onebit tokenizer_replace_meta_test) # A SentencePiece-shaped vocabulary that KEEPS its merge list (engine#1189): the shape Phi-3.5-mini's # pack has, whose ids come from Hugging Face `tokenizers` on the same fixture. No model needed. add_test(NAME tokenizer_spm_bpe @@ -1690,6 +1696,14 @@ if(ONEBIT_NPU_MODEL) add_test(NAME tokenizer_qwen3 COMMAND tokenizer_test ${ONEBIT_NPU_MODEL}/tokenizer.json ${CMAKE_SOURCE_DIR}/tests/data/qwen3_tokenizer_golden.json) endif() +# The Gemma family's own tokenizer.json through the NPU reader (engine#2215), against the reference-library cases +# already kept for it under tests/data/tokenizers/. The file is 32 MB and is not in the tree, so this needs +# -DONEBIT_GEMMA_TOKENIZER= (sha256 cc8d3a0ce364..., the one the cases were made from). +set(ONEBIT_GEMMA_TOKENIZER "" CACHE FILEPATH "A Gemma4 tokenizer.json, for the NPU reader's reference cases") +if(ONEBIT_GEMMA_TOKENIZER) + add_test(NAME tokenizer_gemma4 + COMMAND tokenizer_test ${ONEBIT_GEMMA_TOKENIZER} ${CMAKE_SOURCE_DIR}/tests/data/tokenizers/Gemma4-E4B-IT-NPU2.json) +endif() if(ONEBIT_NPU_MODEL AND ONEBIT_NPU_BO_DUMP) add_test(NAME npu_pack_model COMMAND npu_pack_test ${ONEBIT_NPU_MODEL} ${ONEBIT_NPU_BO_DUMP}) endif() diff --git a/docs/npu.md b/docs/npu.md index d16c579a4..3fc617f8b 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -68,6 +68,14 @@ curl -s http://127.0.0.1:8000/v1/chat/completions -H 'Content-Type: application/ npu/ the model's NPU kernels ``` +The NPU route reads `tokenizer.json` with its own reader (`npu/tokenizer.cpp`). Four vocabulary shapes are handled: +byte-level BPE; SentencePiece pieces with no merges; SentencePiece pieces with merges and a normalizer that prepends +the word-boundary symbol (Phi-3.5); and BPE with merges whose normalizer only rewrites spaces to that symbol and +prepends none (the Gemma family, engine#2215). The last one is encoded by rank over the whole run with no prefix, and +the symbol decodes back to a space. Before that shape was recognised a Gemma4 prompt encoded to 31 ids where the +reference library gives 21, and replies kept the raw symbols. With `-DONEBIT_GEMMA_TOKENIZER=` ctest +`tokenizer_gemma4` holds the reader to the 17 reference cases kept for that file (2 of 17 before, 17 of 17 now). + **What ends an answer.** The serve stops on every id in `config.json`'s `eos_token_id` (one integer or a list), on `<|endoftext|>` when the tokenizer has it, on `tokenizer_config.json`'s `eos_token` when that file is beside the tokenizer, and on the marker the chat route's prompts close an assistant turn with: `<|im_end|>` on the ChatML diff --git a/npu/tokenizer.cpp b/npu/tokenizer.cpp index 77055c66a..2041beb49 100644 --- a/npu/tokenizer.cpp +++ b/npu/tokenizer.cpp @@ -81,6 +81,21 @@ bool spm_prepends_meta(const nlohmann::json& n) { return false; } +// True when the normalizer rewrites every space to the metaspace symbol and does NOT prepend one: a +// {Replace " " -> "\u2581"} step, bare or inside a Sequence, with no Prepend beside it. Gemma4's tokenizer.json is +// this shape (BPE with merges, this normalizer, a decoder that turns the symbol back into a space). After that +// normalizer there is no space left for the pre_tokenizer to split on, so a run of text is one word, merged by rank. +bool replaces_space_with_meta(const nlohmann::json& n) { + if (!n.is_object()) return false; + if (n.value("type", "") == "Replace" && n.contains("pattern") && n["pattern"].is_object() && + n["pattern"].value("String", "") == " ") + return n.value("content", "") == "\xE2\x96\x81"; + if (n.contains("normalizers")) + for (const auto& sub : n["normalizers"]) + if (replaces_space_with_meta(sub)) return true; + return false; +} + // The pattern from tokenizer.json's pre_tokenizer (a Split with a Regex), if any. std::string split_pattern(const nlohmann::json& pre) { if (!pre.is_object()) return ""; @@ -226,8 +241,16 @@ Tokenizer::Tokenizer(const std::string& path) { // is" -- engine#1189. Measured on that pack's own vocabulary, greedy longest-match is not a // substitute for the merges either (it disagrees with the reference on 6 of 20 strings), so // a vocabulary that keeps its merges is merged by rank -- see spm_merged(). - spm_ = has_meta && (merges.empty() || (root.contains("normalizer") && spm_prepends_meta(root["normalizer"]))); + const bool prepends = root.contains("normalizer") && spm_prepends_meta(root["normalizer"]); + // A vocabulary with merges whose normalizer only rewrites spaces (no Prepend): the same pieces and the same + // rank merge, without the dummy prefix. It used to fall to the byte-level path, which split on spaces and + // byte-mapped them: a Gemma4 prompt came out as 31 ids where the reference gives 21, and a reply came back + // with the metaspace symbols still in it. + const bool replaces = !prepends && !merges.empty() && root.contains("normalizer") && + replaces_space_with_meta(root["normalizer"]); + spm_ = has_meta && (merges.empty() || prepends || replaces); spm_merges_ = spm_ && !merges.empty(); + spm_no_prefix_ = spm_ && replaces; if (spm_) { for (const auto& [text, id] : vocab_) { if (text.size() == 6 && text.compare(0, 3, "<0x") == 0 && text[5] == '>') { @@ -317,7 +340,7 @@ std::vector Tokenizer::spm_merged(const std::string& text) const { static const char kMeta[] = "\xE2\x96\x81"; std::string s; s.reserve(text.size() + 3); - s += kMeta; + if (!spm_no_prefix_) s += kMeta; for (char c : text) { if (c == ' ') s += kMeta; else s += c; @@ -440,7 +463,7 @@ std::string Tokenizer::decode(const std::vector& ids, bool strip_dummy_pref // Without this every decoded string came back with a leading space // (MiniCPM4: "The capital of France is" -> " The capital of France is"), // which is visible in 1bit serve's output. - if (strip_dummy_prefix && !out.empty() && out[0] == ' ') out.erase(0, 1); + if (strip_dummy_prefix && !spm_no_prefix_ && !out.empty() && out[0] == ' ') out.erase(0, 1); return out; } std::string out; diff --git a/npu/tokenizer.h b/npu/tokenizer.h index 90cac20b8..6b937b978 100644 --- a/npu/tokenizer.h +++ b/npu/tokenizer.h @@ -61,6 +61,9 @@ class Tokenizer { // still needs sentencepiece rules -- Phi-3.5, engine#1189), then encoded with sentencepiece // rules instead of byte-level BPE (which would emit one id per byte). bool spm_ = false; + // The normalizer rewrites spaces to the metaspace symbol but prepends none (Gemma4): no dummy prefix is added on + // encode and none is stripped on decode. + bool spm_no_prefix_ = false; // SentencePiece AND a merge list (Phi-3.5, engine#1189): pieces are merged by rank over the // normalised text instead of greedy longest-match, which is only right for a merge-less // vocabulary. diff --git a/tests/data/tokenizer_replace_meta.json b/tests/data/tokenizer_replace_meta.json new file mode 100644 index 000000000..90d7bbb9d --- /dev/null +++ b/tests/data/tokenizer_replace_meta.json @@ -0,0 +1,88 @@ +{ + "version": "1.0", + "truncation": null, + "padding": null, + "added_tokens": [ + { + "id": 8, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + } + ], + "normalizer": { + "type": "Replace", + "pattern": { + "String": " " + }, + "content": "▁" + }, + "pre_tokenizer": { + "type": "Split", + "pattern": { + "String": " " + }, + "behavior": "MergedWithPrevious", + "invert": false + }, + "post_processor": null, + "decoder": { + "type": "Sequence", + "decoders": [ + { + "type": "Replace", + "pattern": { + "String": "▁" + }, + "content": " " + }, + { + "type": "ByteFallback" + }, + { + "type": "Fuse" + } + ] + }, + "model": { + "type": "BPE", + "dropout": null, + "unk_token": null, + "continuing_subword_prefix": null, + "end_of_word_suffix": null, + "fuse_unk": false, + "byte_fallback": false, + "ignore_merges": false, + "vocab": { + "▁": 0, + "a": 1, + "b": 2, + "▁a": 3, + "▁b": 4, + "ab": 5, + "\n": 6, + "▁ab": 7 + }, + "merges": [ + [ + "▁", + "a" + ], + [ + "▁", + "b" + ], + [ + "a", + "b" + ], + [ + "▁", + "ab" + ] + ] + } +} diff --git a/tests/tokenizer_replace_meta_test.cpp b/tests/tokenizer_replace_meta_test.cpp new file mode 100644 index 000000000..64519d536 --- /dev/null +++ b/tests/tokenizer_replace_meta_test.cpp @@ -0,0 +1,73 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// tokenizer_replace_meta_test +// +// A BPE vocabulary with merges whose normalizer rewrites every space to the metaspace symbol and prepends none, and +// whose decoder turns the symbol back into a space. Gemma4's tokenizer.json is this shape. The engine used to send +// it down the byte-level path: a Gemma4 prompt came out as 31 ids where the reference gives 21, and a reply came +// back with the symbols still in it. +// +// tests/data/tokenizer_replace_meta.json is a nine-token vocabulary of that shape. The expected ids below are what +// the Hugging Face `tokenizers` library returns for the same file (0.23.3), not what this code returns. + +#include +#include +#include + +#include "tokenizer.h" + +int main(int argc, char** argv) { + if (argc != 2) { + std::fprintf(stderr, "usage: tokenizer_replace_meta_test \n"); + return 2; + } + const onebit::npu::Tokenizer tok(argv[1]); + struct Case { + const char* what; + std::string text; + std::vector want; + }; + const std::vector cases = { + {"a space joins the word after it, and nothing is prepended", "a b", {1, 4}}, + {"merges go by rank across the whole run, not word by word", "ab ab", {5, 3, 2}}, + {"a leading space is a real space", " a", {3}}, + {"a newline is its own piece", "a\nb", {1, 6, 2}}, + {"an added token splits the run", "a b", {8, 1, 4}}, + {"two spaces are two symbols", "a b", {1, 0, 4}}, + {"one letter", "b", {2}}, + // every case decodes back to its text; this one has all three things a decode can lose at once + {"a leading space, two spaces and a newline survive the round trip", " a b\na", {3, 0, 4, 6, 1}}, + }; + int fail = 0; + for (const Case& c : cases) { + const std::vector got = tok.encode(c.text); + const std::string back = tok.decode(got); + const bool ok = got == c.want && back == c.text; + std::printf("%s %s\n", ok ? "ok " : "FAIL", c.what); + if (!ok) { + fail++; + std::printf(" ids:"); + for (int i : got) std::printf(" %d", i); + std::printf(" decoded: [%s]\n", back.c_str()); + } + } + // One token of a stream: the symbol becomes a space there too. + if (tok.decode(4) != " b") { std::printf("FAIL a single streamed token keeps its space: [%s]\n", tok.decode(4).c_str()); fail++; } + else std::printf("ok a single streamed token keeps its space\n"); + if (fail) { std::printf("tokenizer_replace_meta: %d FAILED\n", fail); return 1; } + std::printf("tokenizer_replace_meta: ok\n"); + return 0; +}