diff --git a/app/forward_serve.cpp b/app/forward_serve.cpp index 4a43ab585..0ead20865 100644 --- a/app/forward_serve.cpp +++ b/app/forward_serve.cpp @@ -1088,6 +1088,7 @@ int run_forward_serve(int argc, char** argv) { e.bos_id = json_int(cfg, "bos_token_id", -1); e.add_bos = json_int(cfg, "add_bos_token", 0) != 0; e.eos = npu::eos_token_ids_from_config(cfg); + const size_t eos_from_config = e.eos.size(); e.endoftext = e.tok->token_id("<|endoftext|>"); // q4nx_config_int finds the key at any depth (a text_config's too). e.max_context = q4nx_config_int(cfg.c_str(), "max_position_embeddings", 0); @@ -1143,6 +1144,20 @@ int run_forward_serve(int argc, char** argv) { (specials.empty() ? std::string() : " (model specials: " + specials + ")"); std::fprintf(stderr, "1bit forward-serve: chat route disabled: %s\n", e.chat_refusal.c_str()); } + // What ends a generation is more than config.json's eos_token_id (npu::stop_token_ids): the tokenizer's own + // eos_token, and, when this route closes its prompt turns with ChatML's marker, that marker. Decided here, + // after the template, because whether the route renders ChatML is known only now. + { + const bool chatml_route = !can_render && e.chat_ok; + e.eos = npu::stop_token_ids(npu::read_json_or_null(cfg), npu::read_json_or_null(dir + "/tokenizer_config.json"), + chatml_route ? std::string("<|im_end|>") : std::string(), + [&](const std::string& text) { return e.tok->token_id(text); }); + std::string named; + for (size_t i = 0; i < e.eos.size(); i++) + named += std::string(i ? ", " : "") + std::to_string(e.eos[i]) + " " + e.tok->decode(e.eos[i]) + + (i < eos_from_config ? "" : " (tokenizer / chat turn end)"); + std::fprintf(stderr, "1bit forward-serve: stops on %s\n", named.empty() ? "<|endoftext|> only" : named.c_str()); + } // The template variables `{{ bos_token }}`/`{{ eos_token }}` render: the strings the tokenizer // actually has (a null bos_token in tokenizer_config.json is "", and the engine's -1 is the same // fact), not a second read of the JSON that could disagree with the tokenizer. diff --git a/app/npu_stop_tokens.h b/app/npu_stop_tokens.h index aa910004c..d3fe3c3f7 100644 --- a/app/npu_stop_tokens.h +++ b/app/npu_stop_tokens.h @@ -56,6 +56,61 @@ inline std::vector eos_token_ids(const nlohmann::json& cfg) { return ids; } +/// tokenizer_config.json's `eos_token` as text: a plain string, or the AddedToken object form +/// `{"content": "<|im_end|>", ...}`. Empty when the key is absent, null or another shape. +inline std::string eos_token_text(const nlohmann::json& tokenizer_config) { + if (!tokenizer_config.is_object() || !tokenizer_config.contains("eos_token")) return {}; + const nlohmann::json& e = tokenizer_config["eos_token"]; + if (e.is_string()) return e.get(); + if (e.is_object() && e.contains("content") && e["content"].is_string()) return e["content"].get(); + return {}; +} + +/// Every id that ends a generation on a container, in the order they were found and without repeats: +/// +/// 1. config.json's `eos_token_id` (one integer or a list): what eos_token_ids() reads; +/// 2. tokenizer_config.json's `eos_token`, looked up in the tokenizer. The two files are written by +/// different tools and need not agree. FastFlowLM's Qwen3.5-2B container says `eos_token_id: 248044` +/// (<|endoftext|>) in config.json and `eos_token: "<|im_end|>"` (248046) in tokenizer_config.json, and the +/// model ends a chat turn with the second. With only the first, the answer came back as +/// "Item 3 is the 3D printer.<|im_end|>\n": the turn end served as text and one more token generated +/// after it (engine#1785's row on that container); +/// 3. `chat_turn_end`, when the caller's chat route renders a prompt whose turns end with a marker of its +/// own (ChatML's <|im_end|>): a prompt we close with that marker is answered by a turn that ends with it, +/// whatever the two files say. Pass an empty string when the route has no such marker. +/// +/// `id_of` maps a token's text to its id, or a negative number when the tokenizer has no such token; a name it +/// does not know adds nothing. +template +std::vector stop_token_ids(const nlohmann::json& config, const nlohmann::json& tokenizer_config, + const std::string& chat_turn_end, IdOf&& id_of) { + std::vector ids = eos_token_ids(config); + const auto add = [&](const std::string& text) { + if (text.empty()) return; + const int id = id_of(text); + if (id < 0) return; + for (int have : ids) + if (have == id) return; + ids.push_back(id); + }; + add(eos_token_text(tokenizer_config)); + add(chat_turn_end); + return ids; +} + +/// A JSON file, or null when it is missing or malformed: both stop-id sources are optional. +inline nlohmann::json read_json_or_null(const std::string& path) { + std::ifstream f(path); + if (!f) return nullptr; + nlohmann::json j; + try { + f >> j; + } catch (const std::exception&) { + return nullptr; + } + return j; +} + /// eos_token_ids() for a container's config.json. Unreadable file -> empty. inline std::vector eos_token_ids_from_config(const std::string& cfg_path) { std::ifstream f(cfg_path); diff --git a/docs/npu.md b/docs/npu.md index 3ace1d57f..71ac34ff1 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -68,6 +68,12 @@ curl -s http://127.0.0.1:8000/v1/chat/completions -H 'Content-Type: application/ npu/ the model's NPU kernels ``` +**What ends an answer.** The serve stops on every id in `config.json`'s `eos_token_id` (one integer or a list), on +`<|endoftext|>` when the tokenizer has it, on `tokenizer_config.json`'s `eos_token` when that file is beside the +tokenizer, and, when the chat route falls back to ChatML, on `<|im_end|>`. The two files need not agree: FastFlowLM's +`Qwen3.5-2B-NPU2` container names `<|endoftext|>` in the first and `<|im_end|>` in the second, and the model ends a +chat turn with the second. The load prints the list (`forward-serve: stops on …`). + `1bit serve -m ` serves such a directory on the NPU behind the OpenAI-compatible API ([serve.md](serve.md)); inside Lemonade, that is what its `onebit` backend runs (`--onebit npu`). diff --git a/tests/npu_stop_tokens_test.cpp b/tests/npu_stop_tokens_test.cpp index 3779d786f..6ff5c20ae 100644 --- a/tests/npu_stop_tokens_test.cpp +++ b/tests/npu_stop_tokens_test.cpp @@ -79,6 +79,48 @@ int main() { "unreadable config"); std::remove(cfg.c_str()); + // 6. the tokenizer's eos_token and the chat route's turn end (engine#1785, FastFlowLM's Qwen3.5-2B container): + // config.json names <|endoftext|>, tokenizer_config.json names <|im_end|>, and the model ends its turn with + // the second, which was served as text. + { + using onebit::npu::eos_token_text; + using onebit::npu::stop_token_ids; + const auto id_of = [](const std::string& t) { + if (t == "<|endoftext|>") return 248044; + if (t == "<|im_end|>") return 248046; + if (t == "") return 106; + return -1; + }; + const json cfg = json::parse(R"({"eos_token_id": 248044})"); + const json tok = json::parse(R"({"eos_token": "<|im_end|>"})"); + check(eos_token_text(tok) == "<|im_end|>", "eos_token as a plain string"); + check(eos_token_text(json::parse(R"({"eos_token": {"content": "<|im_end|>", "lstrip": false}})")) == "<|im_end|>", + "eos_token as an AddedToken object"); + check(eos_token_text(json::parse(R"({"eos_token": null})")).empty() && eos_token_text(json::parse("{}")).empty() && + eos_token_text(json::parse(R"({"eos_token": 7})")).empty() && eos_token_text(json(nullptr)).empty(), + "a missing, null or non-text eos_token is no token"); + check_ids(stop_token_ids(cfg, tok, "", id_of), {248044, 248046}, + "the tokenizer's eos_token is a stop id beside config.json's (the token that was served as text)"); + check_ids(stop_token_ids(cfg, json(nullptr), "", id_of), {248044}, "with no tokenizer_config the list is config.json's, unchanged"); + check_ids(stop_token_ids(cfg, json(nullptr), "<|im_end|>", id_of), {248044, 248046}, + "a route that closes its turns with <|im_end|> stops on it even when neither file names it"); + check_ids(stop_token_ids(cfg, tok, "<|im_end|>", id_of), {248044, 248046}, "named twice, listed once"); + check_ids(stop_token_ids(json::parse(R"({"eos_token_id": [248046, 248044]})"), tok, "<|im_end|>", id_of), {248046, 248044}, + "config.json's order is kept and nothing is repeated"); + check_ids(stop_token_ids(cfg, json::parse(R"({"eos_token": "<|eot_id|>"})"), "<|nope|>", id_of), {248044}, + "a name the tokenizer does not have adds nothing"); + check_ids(stop_token_ids(json::parse("{}"), json::parse(R"({"eos_token": ""})"), "", id_of), {106}, + "with no eos_token_id at all, the tokenizer's eos_token is the list"); + check_ids(stop_token_ids(json(nullptr), json(nullptr), "", id_of), {}, "nothing readable, nothing added"); + const std::string f = write_cfg("tokcfg", R"({"eos_token": "<|im_end|>"})"); + check(eos_token_text(onebit::npu::read_json_or_null(f)) == "<|im_end|>", "read from a file"); + check(onebit::npu::read_json_or_null("/tmp/npu_stop_tokens_does_not_exist.json").is_null(), "a missing file is null"); + const std::string bad = write_cfg("badjson", "{ not json"); + check(onebit::npu::read_json_or_null(bad).is_null(), "a malformed file is null"); + std::remove(f.c_str()); + std::remove(bad.c_str()); + } + if (failures == 0) std::printf("all stop-token checks passed\n"); return failures == 0 ? 0 : 1; }