From 04cb6bcff00c79c80280e387f207bee12565069d Mon Sep 17 00:00:00 2001 From: pi-prs Date: Wed, 11 Nov 2026 15:13:31 -0400 Subject: [PATCH] npu: a Replace-only normalizer asks for no leading metaspace, so neither path may add or strip one (engine#2215) Gemma's tokenizer.json is a BPE with 514,906 merges whose normalizer is {Replace " " -> U+2581} and whose pre_tokenizer is a Split on " " with MergedWithPrevious -- after the replace there is no space left to split, so the whole text is one pretoken and the pieces merge by rank over the metaspace symbols. That is exactly spm_merged()'s work, except that spm_merged() PREPENDS a metaspace for the whole text, because SentencePiece's add_dummy_prefix synthesises one. The reader keyed on that: spm_ only took a merged vocabulary when the normalizer declared a Prepend (engine#1189's Phi-3.5 shape), so a Replace-only normalizer fell through to plain BPE, which never introduces the metaspace at all -- spaces stayed literal and encoded to byte-fallback ids. So the prepend, not the merging, is what has to differ: spm_ = has_meta && (merges.empty() || prepends || replaces) spm_prepend_meta_ = prepends || merges.empty() and both spm() and spm_merged() honour the flag when building the text. The decoder needs the same distinction from the other side: it strips the first metaspace as a dummy prefix, which is only correct when one was synthesised -- otherwise the first metaspace is a real space, and stripping it turned decode [U+2581] into "" and a two-space lead into one. Measured against the committed goldens, paired by sha256 with the reference (tokenizers 0.23.3) on the same tokenizer.json: Gemma4-E2B-IT-NPU2 before 2/17 after 17/17 Gemma3-1B-NPU2 before 2/17 after 17/17 LFM2-1.2B-NPU2 before 17/17 after 17/17 (regression check) Qwen3-0.6B-NPU2 before 17/17 after 17/17 (regression check) Qwen3-1.7B-NPU2 before 17/17 after 17/17 Qwen3.5-4B-NPU2 before 17/17 after 17/17 The first failing encode case before the fix was the smallest possible input: encode " " wanted [236743] (the metaspace piece) and got [245237], which is the here/is/a/Python output the issue reports. Verified incrementally in an existing build tree (npu/tokenizer.{h,cpp} were identical to main's before this patch): the goldens above are the verification. --- npu/tokenizer.cpp | 34 ++++++++++++++++++++++++++++++---- npu/tokenizer.h | 4 ++++ 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/npu/tokenizer.cpp b/npu/tokenizer.cpp index 77055c66a..43c612efe 100644 --- a/npu/tokenizer.cpp +++ b/npu/tokenizer.cpp @@ -81,6 +81,25 @@ bool spm_prepends_meta(const nlohmann::json& n) { return false; } +// engine#2215: true when the normalizer REPLACES a space with the metaspace symbol without Prepend. +// Gemma's tokenizer.json is exactly this: {"type":"Replace","pattern":{"String":" "},"content":"\u2581"}, +// with 514,906 merges. The pieces are merged by rank like spm_merged(), but the text must not gain a +// LEADING metaspace -- the normalizer never asks for one -- so the prepend is what has to differ. +bool spm_replaces_meta(const nlohmann::json& n) { + if (!n.is_object()) return false; + if (n.value("type", "") == "Replace") { + const bool space = n.contains("pattern") && + (n["pattern"].value("String", "") == " " || + n["pattern"].value("Regex", "") == " "); + const bool meta = n.value("content", "").find("\xE2\x96\x81") != std::string::npos; + if (space && meta) return true; + } + if (n.contains("normalizers")) + for (const auto& sub : n["normalizers"]) + if (spm_replaces_meta(sub)) return true; + return false; +} + // The pattern from tokenizer.json's pre_tokenizer (a Split with a Regex), if any. std::string split_pattern(const nlohmann::json& pre) { if (!pre.is_object()) return ""; @@ -226,8 +245,12 @@ Tokenizer::Tokenizer(const std::string& path) { // is" -- engine#1189. Measured on that pack's own vocabulary, greedy longest-match is not a // substitute for the merges either (it disagrees with the reference on 6 of 20 strings), so // a vocabulary that keeps its merges is merged by rank -- see spm_merged(). - spm_ = has_meta && (merges.empty() || (root.contains("normalizer") && spm_prepends_meta(root["normalizer"]))); + const bool prepends = root.contains("normalizer") && spm_prepends_meta(root["normalizer"]); + const bool replaces = root.contains("normalizer") && spm_replaces_meta(root["normalizer"]); + spm_ = has_meta && (merges.empty() || prepends || replaces); spm_merges_ = spm_ && !merges.empty(); + // a Replace-only normalizer asks for no leading metaspace; every other spm path adds one + spm_prepend_meta_ = prepends || merges.empty(); if (spm_) { for (const auto& [text, id] : vocab_) { if (text.size() == 6 && text.compare(0, 3, "<0x") == 0 && text[5] == '>') { @@ -317,7 +340,7 @@ std::vector Tokenizer::spm_merged(const std::string& text) const { static const char kMeta[] = "\xE2\x96\x81"; std::string s; s.reserve(text.size() + 3); - s += kMeta; + if (spm_prepend_meta_) s += kMeta; for (char c : text) { if (c == ' ') s += kMeta; else s += c; @@ -335,7 +358,7 @@ std::vector Tokenizer::spm(const std::string& text) const { static const char kMeta[] = "\xE2\x96\x81"; std::string s; s.reserve(text.size() + 3); - s += kMeta; + if (spm_prepend_meta_) s += kMeta; for (char c : text) { if (c == ' ') s += kMeta; else s += c; @@ -440,7 +463,10 @@ std::string Tokenizer::decode(const std::vector& ids, bool strip_dummy_pref // Without this every decoded string came back with a leading space // (MiniCPM4: "The capital of France is" -> " The capital of France is"), // which is visible in 1bit serve's output. - if (strip_dummy_prefix && !out.empty() && out[0] == ' ') out.erase(0, 1); + // engine#2215: only strip the dummy prefix when spm() synthesised one. With a Replace-only + // normalizer ({Replace " " -> U+2581}) there is no dummy prefix, so the first metaspace is a + // REAL space: stripping it turned `decode ["\u2581"]` into "" and a two-space lead into one. + if (strip_dummy_prefix && spm_prepend_meta_ && !out.empty() && out[0] == ' ') out.erase(0, 1); return out; } std::string out; diff --git a/npu/tokenizer.h b/npu/tokenizer.h index 90cac20b8..262dc6cf3 100644 --- a/npu/tokenizer.h +++ b/npu/tokenizer.h @@ -65,6 +65,10 @@ class Tokenizer { // normalised text instead of greedy longest-match, which is only right for a merge-less // vocabulary. bool spm_merges_ = false; + // engine#2215: whether the metaspace is prepended for the whole text. A normalizer that only + // REPLACES spaces ({Replace " " -> U+2581}, no Prepend) turns them into metaspace symbols without + // asking for a leading one, so the merged-BPE path must not add it. + bool spm_prepend_meta_ = true; size_t spm_max_piece_ = 0; std::unordered_map byte_fallback_; // byte -> <0xNN> id uint32_t byte_to_char_[256];