From 468a25f69e8b6f62d61de56ac81f75eb985ea574 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Wed, 11 Nov 2026 11:39:52 -0400 Subject: [PATCH 1/2] npu tokenizer: read a vocabulary whose normalizer rewrites spaces to the word-boundary symbol and prepends none (fixes #2215) Gemma-family tokenizer.json files are BPE with merges and a {Replace ' ' -> U+2581} normalizer. They fell to the byte-level path: a Gemma4 prompt encoded to 31 ids for the reference's 21 and replies kept the raw symbols. The shape is now merged by rank over the whole run with no dummy prefix, and the symbol decodes to a space. Co-Authored-By: Claude Fable 5.2 --- CMakeLists.txt | 6 ++ docs/npu.md | 7 ++ npu/tokenizer.cpp | 29 ++++++++- npu/tokenizer.h | 3 + tests/data/tokenizer_replace_meta.json | 88 ++++++++++++++++++++++++++ tests/tokenizer_replace_meta_test.cpp | 71 +++++++++++++++++++++ 6 files changed, 201 insertions(+), 3 deletions(-) create mode 100644 tests/data/tokenizer_replace_meta.json create mode 100644 tests/tokenizer_replace_meta_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 12feff585..12068c905 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1657,6 +1657,12 @@ target_link_libraries(tokenizer_test PRIVATE onebit_npu) add_executable(tokenizer_linear_test tests/tokenizer_linear_test.cpp) target_link_libraries(tokenizer_linear_test PRIVATE onebit_npu) add_test(NAME tokenizer_linear COMMAND tokenizer_linear_test ${CMAKE_SOURCE_DIR}/tests/data/tokenizer_linear.json) +# A vocabulary whose normalizer rewrites spaces to the metaspace symbol without prepending one (Gemma4's shape): +# ids held to the reference library's on a synthetic nine-token file; no model needed, runs in CI. +add_executable(tokenizer_replace_meta_test tests/tokenizer_replace_meta_test.cpp) +target_link_libraries(tokenizer_replace_meta_test PRIVATE onebit_npu) +add_test(NAME tokenizer_replace_meta COMMAND tokenizer_replace_meta_test ${CMAKE_SOURCE_DIR}/tests/data/tokenizer_replace_meta.json) +add_dependencies(onebit tokenizer_replace_meta_test) # A SentencePiece-shaped vocabulary that KEEPS its merge list (engine#1189): the shape Phi-3.5-mini's # pack has, whose ids come from Hugging Face `tokenizers` on the same fixture. No model needed. add_test(NAME tokenizer_spm_bpe diff --git a/docs/npu.md b/docs/npu.md index baedf4d8e..81bf65d5d 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -68,6 +68,13 @@ curl -s http://127.0.0.1:8000/v1/chat/completions -H 'Content-Type: application/ npu/ the model's NPU kernels ``` +The NPU route reads `tokenizer.json` with its own reader (`npu/tokenizer.cpp`). Four vocabulary shapes are handled: +byte-level BPE; SentencePiece pieces with no merges; SentencePiece pieces with merges and a normalizer that prepends +the word-boundary symbol (Phi-3.5); and BPE with merges whose normalizer only rewrites spaces to that symbol and +prepends none (the Gemma family, engine#2215). The last one is encoded by rank over the whole run with no prefix, and +the symbol decodes back to a space. Before that shape was recognised a Gemma4 prompt encoded to 31 ids where the +reference library gives 21, and replies kept the raw symbols. + **What ends an answer.** The serve stops on every id in `config.json`'s `eos_token_id` (one integer or a list), on `<|endoftext|>` when the tokenizer has it, on `tokenizer_config.json`'s `eos_token` when that file is beside the tokenizer, and on the marker the chat route's prompts close an assistant turn with: `<|im_end|>` on the ChatML diff --git a/npu/tokenizer.cpp b/npu/tokenizer.cpp index 77055c66a..2041beb49 100644 --- a/npu/tokenizer.cpp +++ b/npu/tokenizer.cpp @@ -81,6 +81,21 @@ bool spm_prepends_meta(const nlohmann::json& n) { return false; } +// True when the normalizer rewrites every space to the metaspace symbol and does NOT prepend one: a +// {Replace " " -> "\u2581"} step, bare or inside a Sequence, with no Prepend beside it. Gemma4's tokenizer.json is +// this shape (BPE with merges, this normalizer, a decoder that turns the symbol back into a space). After that +// normalizer there is no space left for the pre_tokenizer to split on, so a run of text is one word, merged by rank. +bool replaces_space_with_meta(const nlohmann::json& n) { + if (!n.is_object()) return false; + if (n.value("type", "") == "Replace" && n.contains("pattern") && n["pattern"].is_object() && + n["pattern"].value("String", "") == " ") + return n.value("content", "") == "\xE2\x96\x81"; + if (n.contains("normalizers")) + for (const auto& sub : n["normalizers"]) + if (replaces_space_with_meta(sub)) return true; + return false; +} + // The pattern from tokenizer.json's pre_tokenizer (a Split with a Regex), if any. std::string split_pattern(const nlohmann::json& pre) { if (!pre.is_object()) return ""; @@ -226,8 +241,16 @@ Tokenizer::Tokenizer(const std::string& path) { // is" -- engine#1189. Measured on that pack's own vocabulary, greedy longest-match is not a // substitute for the merges either (it disagrees with the reference on 6 of 20 strings), so // a vocabulary that keeps its merges is merged by rank -- see spm_merged(). - spm_ = has_meta && (merges.empty() || (root.contains("normalizer") && spm_prepends_meta(root["normalizer"]))); + const bool prepends = root.contains("normalizer") && spm_prepends_meta(root["normalizer"]); + // A vocabulary with merges whose normalizer only rewrites spaces (no Prepend): the same pieces and the same + // rank merge, without the dummy prefix. It used to fall to the byte-level path, which split on spaces and + // byte-mapped them: a Gemma4 prompt came out as 31 ids where the reference gives 21, and a reply came back + // with the metaspace symbols still in it. + const bool replaces = !prepends && !merges.empty() && root.contains("normalizer") && + replaces_space_with_meta(root["normalizer"]); + spm_ = has_meta && (merges.empty() || prepends || replaces); spm_merges_ = spm_ && !merges.empty(); + spm_no_prefix_ = spm_ && replaces; if (spm_) { for (const auto& [text, id] : vocab_) { if (text.size() == 6 && text.compare(0, 3, "<0x") == 0 && text[5] == '>') { @@ -317,7 +340,7 @@ std::vector Tokenizer::spm_merged(const std::string& text) const { static const char kMeta[] = "\xE2\x96\x81"; std::string s; s.reserve(text.size() + 3); - s += kMeta; + if (!spm_no_prefix_) s += kMeta; for (char c : text) { if (c == ' ') s += kMeta; else s += c; @@ -440,7 +463,7 @@ std::string Tokenizer::decode(const std::vector& ids, bool strip_dummy_pref // Without this every decoded string came back with a leading space // (MiniCPM4: "The capital of France is" -> " The capital of France is"), // which is visible in 1bit serve's output. - if (strip_dummy_prefix && !out.empty() && out[0] == ' ') out.erase(0, 1); + if (strip_dummy_prefix && !spm_no_prefix_ && !out.empty() && out[0] == ' ') out.erase(0, 1); return out; } std::string out; diff --git a/npu/tokenizer.h b/npu/tokenizer.h index 90cac20b8..6b937b978 100644 --- a/npu/tokenizer.h +++ b/npu/tokenizer.h @@ -61,6 +61,9 @@ class Tokenizer { // still needs sentencepiece rules -- Phi-3.5, engine#1189), then encoded with sentencepiece // rules instead of byte-level BPE (which would emit one id per byte). bool spm_ = false; + // The normalizer rewrites spaces to the metaspace symbol but prepends none (Gemma4): no dummy prefix is added on + // encode and none is stripped on decode. + bool spm_no_prefix_ = false; // SentencePiece AND a merge list (Phi-3.5, engine#1189): pieces are merged by rank over the // normalised text instead of greedy longest-match, which is only right for a merge-less // vocabulary. diff --git a/tests/data/tokenizer_replace_meta.json b/tests/data/tokenizer_replace_meta.json new file mode 100644 index 000000000..90d7bbb9d --- /dev/null +++ b/tests/data/tokenizer_replace_meta.json @@ -0,0 +1,88 @@ +{ + "version": "1.0", + "truncation": null, + "padding": null, + "added_tokens": [ + { + "id": 8, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + } + ], + "normalizer": { + "type": "Replace", + "pattern": { + "String": " " + }, + "content": "▁" + }, + "pre_tokenizer": { + "type": "Split", + "pattern": { + "String": " " + }, + "behavior": "MergedWithPrevious", + "invert": false + }, + "post_processor": null, + "decoder": { + "type": "Sequence", + "decoders": [ + { + "type": "Replace", + "pattern": { + "String": "▁" + }, + "content": " " + }, + { + "type": "ByteFallback" + }, + { + "type": "Fuse" + } + ] + }, + "model": { + "type": "BPE", + "dropout": null, + "unk_token": null, + "continuing_subword_prefix": null, + "end_of_word_suffix": null, + "fuse_unk": false, + "byte_fallback": false, + "ignore_merges": false, + "vocab": { + "▁": 0, + "a": 1, + "b": 2, + "▁a": 3, + "▁b": 4, + "ab": 5, + "\n": 6, + "▁ab": 7 + }, + "merges": [ + [ + "▁", + "a" + ], + [ + "▁", + "b" + ], + [ + "a", + "b" + ], + [ + "▁", + "ab" + ] + ] + } +} diff --git a/tests/tokenizer_replace_meta_test.cpp b/tests/tokenizer_replace_meta_test.cpp new file mode 100644 index 000000000..fb7bffd5c --- /dev/null +++ b/tests/tokenizer_replace_meta_test.cpp @@ -0,0 +1,71 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// tokenizer_replace_meta_test +// +// A BPE vocabulary with merges whose normalizer rewrites every space to the metaspace symbol and prepends none, and +// whose decoder turns the symbol back into a space. Gemma4's tokenizer.json is this shape. The engine used to send +// it down the byte-level path: a Gemma4 prompt came out as 31 ids where the reference gives 21, and a reply came +// back with the symbols still in it. +// +// tests/data/tokenizer_replace_meta.json is a nine-token vocabulary of that shape. The expected ids below are what +// the Hugging Face `tokenizers` library returns for the same file (0.23.3), not what this code returns. + +#include +#include +#include + +#include "tokenizer.h" + +int main(int argc, char** argv) { + if (argc != 2) { + std::fprintf(stderr, "usage: tokenizer_replace_meta_test \n"); + return 2; + } + const onebit::npu::Tokenizer tok(argv[1]); + struct Case { + const char* what; + std::string text; + std::vector want; + }; + const std::vector cases = { + {"a space joins the word after it, and nothing is prepended", "a b", {1, 4}}, + {"merges go by rank across the whole run, not word by word", "ab ab", {5, 3, 2}}, + {"a leading space is a real space", " a", {3}}, + {"a newline is its own piece", "a\nb", {1, 6, 2}}, + {"an added token splits the run", "a b", {8, 1, 4}}, + {"two spaces are two symbols", "a b", {1, 0, 4}}, + {"one letter", "b", {2}}, + }; + int fail = 0; + for (const Case& c : cases) { + const std::vector got = tok.encode(c.text); + const std::string back = tok.decode(got); + const bool ok = got == c.want && back == c.text; + std::printf("%s %s\n", ok ? "ok " : "FAIL", c.what); + if (!ok) { + fail++; + std::printf(" ids:"); + for (int i : got) std::printf(" %d", i); + std::printf(" decoded: [%s]\n", back.c_str()); + } + } + // One token of a stream: the symbol becomes a space there too. + if (tok.decode(4) != " b") { std::printf("FAIL a single streamed token keeps its space: [%s]\n", tok.decode(4).c_str()); fail++; } + else std::printf("ok a single streamed token keeps its space\n"); + if (fail) { std::printf("tokenizer_replace_meta: %d FAILED\n", fail); return 1; } + std::printf("tokenizer_replace_meta: ok\n"); + return 0; +} From 248158ef6a437da34951330c19302f8ab656ccb8 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Wed, 11 Nov 2026 12:03:04 -0400 Subject: [PATCH 2/2] tokenizer: hold the NPU reader to the reference cases kept for Gemma4's own file, and one round-trip arm (refs #2215) Review of 468a25f asked for a golden fixture on Gemma's own tokenizer.json and a decode round trip with a leading space, two spaces and a newline. The cases already exist under tests/data/tokenizers/; ctest tokenizer_gemma4 runs them when -DONEBIT_GEMMA_TOKENIZER names the file (2 of 17 before the fix, 17 of 17 with it). Co-Authored-By: Claude Fable 5.2 --- CMakeLists.txt | 8 ++++++++ docs/npu.md | 3 ++- tests/tokenizer_replace_meta_test.cpp | 2 ++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 12068c905..adc438da3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1671,6 +1671,14 @@ if(ONEBIT_NPU_MODEL) add_test(NAME tokenizer_qwen3 COMMAND tokenizer_test ${ONEBIT_NPU_MODEL}/tokenizer.json ${CMAKE_SOURCE_DIR}/tests/data/qwen3_tokenizer_golden.json) endif() +# The Gemma family's own tokenizer.json through the NPU reader (engine#2215), against the reference-library cases +# already kept for it under tests/data/tokenizers/. The file is 32 MB and is not in the tree, so this needs +# -DONEBIT_GEMMA_TOKENIZER= (sha256 cc8d3a0ce364..., the one the cases were made from). +set(ONEBIT_GEMMA_TOKENIZER "" CACHE FILEPATH "A Gemma4 tokenizer.json, for the NPU reader's reference cases") +if(ONEBIT_GEMMA_TOKENIZER) + add_test(NAME tokenizer_gemma4 + COMMAND tokenizer_test ${ONEBIT_GEMMA_TOKENIZER} ${CMAKE_SOURCE_DIR}/tests/data/tokenizers/Gemma4-E4B-IT-NPU2.json) +endif() if(ONEBIT_NPU_MODEL AND ONEBIT_NPU_BO_DUMP) add_test(NAME npu_pack_model COMMAND npu_pack_test ${ONEBIT_NPU_MODEL} ${ONEBIT_NPU_BO_DUMP}) endif() diff --git a/docs/npu.md b/docs/npu.md index 81bf65d5d..f31231e62 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -73,7 +73,8 @@ byte-level BPE; SentencePiece pieces with no merges; SentencePiece pieces with m the word-boundary symbol (Phi-3.5); and BPE with merges whose normalizer only rewrites spaces to that symbol and prepends none (the Gemma family, engine#2215). The last one is encoded by rank over the whole run with no prefix, and the symbol decodes back to a space. Before that shape was recognised a Gemma4 prompt encoded to 31 ids where the -reference library gives 21, and replies kept the raw symbols. +reference library gives 21, and replies kept the raw symbols. With `-DONEBIT_GEMMA_TOKENIZER=` ctest +`tokenizer_gemma4` holds the reader to the 17 reference cases kept for that file (2 of 17 before, 17 of 17 now). **What ends an answer.** The serve stops on every id in `config.json`'s `eos_token_id` (one integer or a list), on `<|endoftext|>` when the tokenizer has it, on `tokenizer_config.json`'s `eos_token` when that file is beside the diff --git a/tests/tokenizer_replace_meta_test.cpp b/tests/tokenizer_replace_meta_test.cpp index fb7bffd5c..64519d536 100644 --- a/tests/tokenizer_replace_meta_test.cpp +++ b/tests/tokenizer_replace_meta_test.cpp @@ -48,6 +48,8 @@ int main(int argc, char** argv) { {"an added token splits the run", "a b", {8, 1, 4}}, {"two spaces are two symbols", "a b", {1, 0, 4}}, {"one letter", "b", {2}}, + // every case decodes back to its text; this one has all three things a decode can lose at once + {"a leading space, two spaces and a newline survive the round trip", " a b\na", {3, 0, 4, 6, 1}}, }; int fail = 0; for (const Case& c : cases) {