Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1682,6 +1682,12 @@ target_link_libraries(tokenizer_test PRIVATE onebit_npu)
add_executable(tokenizer_linear_test tests/tokenizer_linear_test.cpp)
target_link_libraries(tokenizer_linear_test PRIVATE onebit_npu)
add_test(NAME tokenizer_linear COMMAND tokenizer_linear_test ${CMAKE_SOURCE_DIR}/tests/data/tokenizer_linear.json)
# A vocabulary whose normalizer rewrites spaces to the metaspace symbol without prepending one (Gemma4's shape):
# ids held to the reference library's on a synthetic nine-token file; no model needed, runs in CI.
add_executable(tokenizer_replace_meta_test tests/tokenizer_replace_meta_test.cpp)
target_link_libraries(tokenizer_replace_meta_test PRIVATE onebit_npu)
add_test(NAME tokenizer_replace_meta COMMAND tokenizer_replace_meta_test ${CMAKE_SOURCE_DIR}/tests/data/tokenizer_replace_meta.json)
add_dependencies(onebit tokenizer_replace_meta_test)
# A SentencePiece-shaped vocabulary that KEEPS its merge list (engine#1189): the shape Phi-3.5-mini's
# pack has, whose ids come from Hugging Face `tokenizers` on the same fixture. No model needed.
add_test(NAME tokenizer_spm_bpe
Expand All @@ -1690,6 +1696,14 @@ if(ONEBIT_NPU_MODEL)
add_test(NAME tokenizer_qwen3
COMMAND tokenizer_test ${ONEBIT_NPU_MODEL}/tokenizer.json ${CMAKE_SOURCE_DIR}/tests/data/qwen3_tokenizer_golden.json)
endif()
# The Gemma family's own tokenizer.json through the NPU reader (engine#2215), against the reference-library cases
# already kept for it under tests/data/tokenizers/. The file is 32 MB and is not in the tree, so this needs
# -DONEBIT_GEMMA_TOKENIZER=<a Gemma4 tokenizer.json> (sha256 cc8d3a0ce364..., the one the cases were made from).
set(ONEBIT_GEMMA_TOKENIZER "" CACHE FILEPATH "A Gemma4 tokenizer.json, for the NPU reader's reference cases")
if(ONEBIT_GEMMA_TOKENIZER)
add_test(NAME tokenizer_gemma4
COMMAND tokenizer_test ${ONEBIT_GEMMA_TOKENIZER} ${CMAKE_SOURCE_DIR}/tests/data/tokenizers/Gemma4-E4B-IT-NPU2.json)
endif()
if(ONEBIT_NPU_MODEL AND ONEBIT_NPU_BO_DUMP)
add_test(NAME npu_pack_model COMMAND npu_pack_test ${ONEBIT_NPU_MODEL} ${ONEBIT_NPU_BO_DUMP})
endif()
Expand Down
8 changes: 8 additions & 0 deletions docs/npu.md
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,14 @@ curl -s http://127.0.0.1:8000/v1/chat/completions -H 'Content-Type: application/
npu/ the model's NPU kernels
```

The NPU route reads `tokenizer.json` with its own reader (`npu/tokenizer.cpp`). Four vocabulary shapes are handled:
byte-level BPE; SentencePiece pieces with no merges; SentencePiece pieces with merges and a normalizer that prepends
the word-boundary symbol (Phi-3.5); and BPE with merges whose normalizer only rewrites spaces to that symbol and
prepends none (the Gemma family, engine#2215). The last one is encoded by rank over the whole run with no prefix, and
the symbol decodes back to a space. Before that shape was recognised a Gemma4 prompt encoded to 31 ids where the
reference library gives 21, and replies kept the raw symbols. With `-DONEBIT_GEMMA_TOKENIZER=<tokenizer.json>` ctest
`tokenizer_gemma4` holds the reader to the 17 reference cases kept for that file (2 of 17 before, 17 of 17 now).

**What ends an answer.** The serve stops on every id in `config.json`'s `eos_token_id` (one integer or a list), on
`<|endoftext|>` when the tokenizer has it, on `tokenizer_config.json`'s `eos_token` when that file is beside the
tokenizer, and on the marker the chat route's prompts close an assistant turn with: `<|im_end|>` on the ChatML
Expand Down
29 changes: 26 additions & 3 deletions npu/tokenizer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,21 @@ bool spm_prepends_meta(const nlohmann::json& n) {
return false;
}

// True when the normalizer rewrites every space to the metaspace symbol and does NOT prepend one: a
// {Replace " " -> "\u2581"} step, bare or inside a Sequence, with no Prepend beside it. Gemma4's tokenizer.json is
// this shape (BPE with merges, this normalizer, a decoder that turns the symbol back into a space). After that
// normalizer there is no space left for the pre_tokenizer to split on, so a run of text is one word, merged by rank.
bool replaces_space_with_meta(const nlohmann::json& n) {
if (!n.is_object()) return false;
if (n.value("type", "") == "Replace" && n.contains("pattern") && n["pattern"].is_object() &&
n["pattern"].value("String", "") == " ")
return n.value("content", "") == "\xE2\x96\x81";
if (n.contains("normalizers"))
for (const auto& sub : n["normalizers"])
if (replaces_space_with_meta(sub)) return true;
return false;
}

// The pattern from tokenizer.json's pre_tokenizer (a Split with a Regex), if any.
std::string split_pattern(const nlohmann::json& pre) {
if (!pre.is_object()) return "";
Expand Down Expand Up @@ -226,8 +241,16 @@ Tokenizer::Tokenizer(const std::string& path) {
// is" -- engine#1189. Measured on that pack's own vocabulary, greedy longest-match is not a
// substitute for the merges either (it disagrees with the reference on 6 of 20 strings), so
// a vocabulary that keeps its merges is merged by rank -- see spm_merged().
spm_ = has_meta && (merges.empty() || (root.contains("normalizer") && spm_prepends_meta(root["normalizer"])));
const bool prepends = root.contains("normalizer") && spm_prepends_meta(root["normalizer"]);
// A vocabulary with merges whose normalizer only rewrites spaces (no Prepend): the same pieces and the same
// rank merge, without the dummy prefix. It used to fall to the byte-level path, which split on spaces and
// byte-mapped them: a Gemma4 prompt came out as 31 ids where the reference gives 21, and a reply came back
// with the metaspace symbols still in it.
const bool replaces = !prepends && !merges.empty() && root.contains("normalizer") &&
replaces_space_with_meta(root["normalizer"]);
spm_ = has_meta && (merges.empty() || prepends || replaces);
spm_merges_ = spm_ && !merges.empty();
spm_no_prefix_ = spm_ && replaces;
if (spm_) {
for (const auto& [text, id] : vocab_) {
if (text.size() == 6 && text.compare(0, 3, "<0x") == 0 && text[5] == '>') {
Expand Down Expand Up @@ -317,7 +340,7 @@ std::vector<int> Tokenizer::spm_merged(const std::string& text) const {
static const char kMeta[] = "\xE2\x96\x81";
std::string s;
s.reserve(text.size() + 3);
s += kMeta;
if (!spm_no_prefix_) s += kMeta;
for (char c : text) {
if (c == ' ') s += kMeta;
else s += c;
Expand Down Expand Up @@ -440,7 +463,7 @@ std::string Tokenizer::decode(const std::vector<int>& ids, bool strip_dummy_pref
// Without this every decoded string came back with a leading space
// (MiniCPM4: "The capital of France is" -> " The capital of France is"),
// which is visible in 1bit serve's output.
if (strip_dummy_prefix && !out.empty() && out[0] == ' ') out.erase(0, 1);
if (strip_dummy_prefix && !spm_no_prefix_ && !out.empty() && out[0] == ' ') out.erase(0, 1);
return out;
}
std::string out;
Expand Down
3 changes: 3 additions & 0 deletions npu/tokenizer.h
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,9 @@ class Tokenizer {
// still needs sentencepiece rules -- Phi-3.5, engine#1189), then encoded with sentencepiece
// rules instead of byte-level BPE (which would emit one id per byte).
bool spm_ = false;
// The normalizer rewrites spaces to the metaspace symbol but prepends none (Gemma4): no dummy prefix is added on
// encode and none is stripped on decode.
bool spm_no_prefix_ = false;
// SentencePiece AND a merge list (Phi-3.5, engine#1189): pieces are merged by rank over the
// normalised text instead of greedy longest-match, which is only right for a merge-less
// vocabulary.
Expand Down
88 changes: 88 additions & 0 deletions tests/data/tokenizer_replace_meta.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 8,
"content": "<bos>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": {
"type": "Replace",
"pattern": {
"String": " "
},
"content": "▁"
},
"pre_tokenizer": {
"type": "Split",
"pattern": {
"String": " "
},
"behavior": "MergedWithPrevious",
"invert": false
},
"post_processor": null,
"decoder": {
"type": "Sequence",
"decoders": [
{
"type": "Replace",
"pattern": {
"String": "▁"
},
"content": " "
},
{
"type": "ByteFallback"
},
{
"type": "Fuse"
}
]
},
"model": {
"type": "BPE",
"dropout": null,
"unk_token": null,
"continuing_subword_prefix": null,
"end_of_word_suffix": null,
"fuse_unk": false,
"byte_fallback": false,
"ignore_merges": false,
"vocab": {
"▁": 0,
"a": 1,
"b": 2,
"▁a": 3,
"▁b": 4,
"ab": 5,
"\n": 6,
"▁ab": 7
},
"merges": [
[
"▁",
"a"
],
[
"▁",
"b"
],
[
"a",
"b"
],
[
"▁",
"ab"
]
]
}
}
73 changes: 73 additions & 0 deletions tests/tokenizer_replace_meta_test.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
// Copyright 2026 bong-water-water-bong
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// tokenizer_replace_meta_test <tokenizer_replace_meta.json>
//
// A BPE vocabulary with merges whose normalizer rewrites every space to the metaspace symbol and prepends none, and
// whose decoder turns the symbol back into a space. Gemma4's tokenizer.json is this shape. The engine used to send
// it down the byte-level path: a Gemma4 prompt came out as 31 ids where the reference gives 21, and a reply came
// back with the symbols still in it.
//
// tests/data/tokenizer_replace_meta.json is a nine-token vocabulary of that shape. The expected ids below are what
// the Hugging Face `tokenizers` library returns for the same file (0.23.3), not what this code returns.

#include <cstdio>
#include <string>
#include <vector>

#include "tokenizer.h"

int main(int argc, char** argv) {
if (argc != 2) {
std::fprintf(stderr, "usage: tokenizer_replace_meta_test <tokenizer_replace_meta.json>\n");
return 2;
}
const onebit::npu::Tokenizer tok(argv[1]);
struct Case {
const char* what;
std::string text;
std::vector<int> want;
};
const std::vector<Case> cases = {
{"a space joins the word after it, and nothing is prepended", "a b", {1, 4}},
{"merges go by rank across the whole run, not word by word", "ab ab", {5, 3, 2}},
{"a leading space is a real space", " a", {3}},
{"a newline is its own piece", "a\nb", {1, 6, 2}},
{"an added token splits the run", "<bos>a b", {8, 1, 4}},
{"two spaces are two symbols", "a b", {1, 0, 4}},
{"one letter", "b", {2}},
// every case decodes back to its text; this one has all three things a decode can lose at once
{"a leading space, two spaces and a newline survive the round trip", " a b\na", {3, 0, 4, 6, 1}},
};
int fail = 0;
for (const Case& c : cases) {
const std::vector<int> got = tok.encode(c.text);
const std::string back = tok.decode(got);
const bool ok = got == c.want && back == c.text;
std::printf("%s %s\n", ok ? "ok " : "FAIL", c.what);
if (!ok) {
fail++;
std::printf(" ids:");
for (int i : got) std::printf(" %d", i);
std::printf(" decoded: [%s]\n", back.c_str());
}
}
// One token of a stream: the symbol becomes a space there too.
if (tok.decode(4) != " b") { std::printf("FAIL a single streamed token keeps its space: [%s]\n", tok.decode(4).c_str()); fail++; }
else std::printf("ok a single streamed token keeps its space\n");
if (fail) { std::printf("tokenizer_replace_meta: %d FAILED\n", fail); return 1; }
std::printf("tokenizer_replace_meta: ok\n");
return 0;
}
Loading