Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions app/forward_serve.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1088,6 +1088,7 @@ int run_forward_serve(int argc, char** argv) {
e.bos_id = json_int(cfg, "bos_token_id", -1);
e.add_bos = json_int(cfg, "add_bos_token", 0) != 0;
e.eos = npu::eos_token_ids_from_config(cfg);
const size_t eos_from_config = e.eos.size();
e.endoftext = e.tok->token_id("<|endoftext|>");
// q4nx_config_int finds the key at any depth (a text_config's too).
e.max_context = q4nx_config_int(cfg.c_str(), "max_position_embeddings", 0);
Expand Down Expand Up @@ -1143,6 +1144,20 @@ int run_forward_serve(int argc, char** argv) {
(specials.empty() ? std::string() : " (model specials: " + specials + ")");
std::fprintf(stderr, "1bit forward-serve: chat route disabled: %s\n", e.chat_refusal.c_str());
}
// What ends a generation is more than config.json's eos_token_id (npu::stop_token_ids): the tokenizer's own
// eos_token, and, when this route closes its prompt turns with ChatML's marker, that marker. Decided here,
// after the template, because whether the route renders ChatML is known only now.
{
const bool chatml_route = !can_render && e.chat_ok;
e.eos = npu::stop_token_ids(npu::read_json_or_null(cfg), npu::read_json_or_null(dir + "/tokenizer_config.json"),
chatml_route ? std::string("<|im_end|>") : std::string(),
[&](const std::string& text) { return e.tok->token_id(text); });
std::string named;
for (size_t i = 0; i < e.eos.size(); i++)
named += std::string(i ? ", " : "") + std::to_string(e.eos[i]) + " " + e.tok->decode(e.eos[i]) +
(i < eos_from_config ? "" : " (tokenizer / chat turn end)");
std::fprintf(stderr, "1bit forward-serve: stops on %s\n", named.empty() ? "<|endoftext|> only" : named.c_str());
}
// The template variables `{{ bos_token }}`/`{{ eos_token }}` render: the strings the tokenizer
// actually has (a null bos_token in tokenizer_config.json is "", and the engine's -1 is the same
// fact), not a second read of the JSON that could disagree with the tokenizer.
Expand Down
55 changes: 55 additions & 0 deletions app/npu_stop_tokens.h
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,61 @@ inline std::vector<int> eos_token_ids(const nlohmann::json& cfg) {
return ids;
}

/// tokenizer_config.json's `eos_token` as text: a plain string, or the AddedToken object form
/// `{"content": "<|im_end|>", ...}`. Empty when the key is absent, null or another shape.
inline std::string eos_token_text(const nlohmann::json& tokenizer_config) {
if (!tokenizer_config.is_object() || !tokenizer_config.contains("eos_token")) return {};
const nlohmann::json& e = tokenizer_config["eos_token"];
if (e.is_string()) return e.get<std::string>();
if (e.is_object() && e.contains("content") && e["content"].is_string()) return e["content"].get<std::string>();
return {};
}

/// Every id that ends a generation on a container, in the order they were found and without repeats:
///
/// 1. config.json's `eos_token_id` (one integer or a list): what eos_token_ids() reads;
/// 2. tokenizer_config.json's `eos_token`, looked up in the tokenizer. The two files are written by
/// different tools and need not agree. FastFlowLM's Qwen3.5-2B container says `eos_token_id: 248044`
/// (<|endoftext|>) in config.json and `eos_token: "<|im_end|>"` (248046) in tokenizer_config.json, and the
/// model ends a chat turn with the second. With only the first, the answer came back as
/// "Item 3 is the 3D printer.<|im_end|>\n": the turn end served as text and one more token generated
/// after it (engine#1785's row on that container);
/// 3. `chat_turn_end`, when the caller's chat route renders a prompt whose turns end with a marker of its
/// own (ChatML's <|im_end|>): a prompt we close with that marker is answered by a turn that ends with it,
/// whatever the two files say. Pass an empty string when the route has no such marker.
///
/// `id_of` maps a token's text to its id, or a negative number when the tokenizer has no such token; a name it
/// does not know adds nothing.
template <typename IdOf>
std::vector<int> stop_token_ids(const nlohmann::json& config, const nlohmann::json& tokenizer_config,
const std::string& chat_turn_end, IdOf&& id_of) {
std::vector<int> ids = eos_token_ids(config);
const auto add = [&](const std::string& text) {
if (text.empty()) return;
const int id = id_of(text);
if (id < 0) return;
for (int have : ids)
if (have == id) return;
ids.push_back(id);
};
add(eos_token_text(tokenizer_config));
add(chat_turn_end);
return ids;
}

/// A JSON file, or null when it is missing or malformed: both stop-id sources are optional.
inline nlohmann::json read_json_or_null(const std::string& path) {
std::ifstream f(path);
if (!f) return nullptr;
nlohmann::json j;
try {
f >> j;
} catch (const std::exception&) {
return nullptr;
}
return j;
}

/// eos_token_ids() for a container's config.json. Unreadable file -> empty.
inline std::vector<int> eos_token_ids_from_config(const std::string& cfg_path) {
std::ifstream f(cfg_path);
Expand Down
6 changes: 6 additions & 0 deletions docs/npu.md
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,12 @@ curl -s http://127.0.0.1:8000/v1/chat/completions -H 'Content-Type: application/
npu/ the model's NPU kernels
```

**What ends an answer.** The serve stops on every id in `config.json`'s `eos_token_id` (one integer or a list), on
`<|endoftext|>` when the tokenizer has it, on `tokenizer_config.json`'s `eos_token` when that file is beside the
tokenizer, and, when the chat route falls back to ChatML, on `<|im_end|>`. The two files need not agree: FastFlowLM's
`Qwen3.5-2B-NPU2` container names `<|endoftext|>` in the first and `<|im_end|>` in the second, and the model ends a
chat turn with the second. The load prints the list (`forward-serve: stops on …`).

`1bit serve -m <model dir>` serves such a directory on the NPU behind the OpenAI-compatible API
([serve.md](serve.md)); inside Lemonade, that is what its `onebit` backend runs (`--onebit npu`).

Expand Down
42 changes: 42 additions & 0 deletions tests/npu_stop_tokens_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,48 @@ int main() {
"unreadable config");
std::remove(cfg.c_str());

// 6. the tokenizer's eos_token and the chat route's turn end (engine#1785, FastFlowLM's Qwen3.5-2B container):
// config.json names <|endoftext|>, tokenizer_config.json names <|im_end|>, and the model ends its turn with
// the second, which was served as text.
{
using onebit::npu::eos_token_text;
using onebit::npu::stop_token_ids;
const auto id_of = [](const std::string& t) {
if (t == "<|endoftext|>") return 248044;
if (t == "<|im_end|>") return 248046;
if (t == "<end_of_turn>") return 106;
return -1;
};
const json cfg = json::parse(R"({"eos_token_id": 248044})");
const json tok = json::parse(R"({"eos_token": "<|im_end|>"})");
check(eos_token_text(tok) == "<|im_end|>", "eos_token as a plain string");
check(eos_token_text(json::parse(R"({"eos_token": {"content": "<|im_end|>", "lstrip": false}})")) == "<|im_end|>",
"eos_token as an AddedToken object");
check(eos_token_text(json::parse(R"({"eos_token": null})")).empty() && eos_token_text(json::parse("{}")).empty() &&
eos_token_text(json::parse(R"({"eos_token": 7})")).empty() && eos_token_text(json(nullptr)).empty(),
"a missing, null or non-text eos_token is no token");
check_ids(stop_token_ids(cfg, tok, "", id_of), {248044, 248046},
"the tokenizer's eos_token is a stop id beside config.json's (the token that was served as text)");
check_ids(stop_token_ids(cfg, json(nullptr), "", id_of), {248044}, "with no tokenizer_config the list is config.json's, unchanged");
check_ids(stop_token_ids(cfg, json(nullptr), "<|im_end|>", id_of), {248044, 248046},
"a route that closes its turns with <|im_end|> stops on it even when neither file names it");
check_ids(stop_token_ids(cfg, tok, "<|im_end|>", id_of), {248044, 248046}, "named twice, listed once");
check_ids(stop_token_ids(json::parse(R"({"eos_token_id": [248046, 248044]})"), tok, "<|im_end|>", id_of), {248046, 248044},
"config.json's order is kept and nothing is repeated");
check_ids(stop_token_ids(cfg, json::parse(R"({"eos_token": "<|eot_id|>"})"), "<|nope|>", id_of), {248044},
"a name the tokenizer does not have adds nothing");
check_ids(stop_token_ids(json::parse("{}"), json::parse(R"({"eos_token": "<end_of_turn>"})"), "", id_of), {106},
"with no eos_token_id at all, the tokenizer's eos_token is the list");
check_ids(stop_token_ids(json(nullptr), json(nullptr), "", id_of), {}, "nothing readable, nothing added");
const std::string f = write_cfg("tokcfg", R"({"eos_token": "<|im_end|>"})");
check(eos_token_text(onebit::npu::read_json_or_null(f)) == "<|im_end|>", "read from a file");
check(onebit::npu::read_json_or_null("/tmp/npu_stop_tokens_does_not_exist.json").is_null(), "a missing file is null");
const std::string bad = write_cfg("badjson", "{ not json");
check(onebit::npu::read_json_or_null(bad).is_null(), "a malformed file is null");
std::remove(f.c_str());
std::remove(bad.c_str());
}

if (failures == 0) std::printf("all stop-token checks passed\n");
return failures == 0 ? 0 : 1;
}
Loading