Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,13 @@ jobs:
run: |
git fetch -q --depth=1 origin "$BASE"
python3 tools/check_pins.py "$BASE"
# engine #963 option 2: the NPU chat route renders the container's own chat template with the pinned
# llama.cpp fork's Jinja engine (app/npu_template.h), so the build needs that submodule's sources —
# shallow, one checkout, ~180 MB, 8 s, and it is the same commit the release build compiles. Without it
# CMake's ONEBIT_JINJA defaults off and `npu_template` would skip, i.e. the renderer would reach a green
# build without ever being compiled.
- name: The llama.cpp sources the chat-template renderer compiles
run: git submodule update --init --depth 1 third_party/llama.cpp
- name: GGUF -> Q4NX repack, ternary -> Q4_0, MTP draft vocabulary
run: |
python3 -m pip install numpy gguf
Expand Down Expand Up @@ -90,6 +97,7 @@ jobs:
laya_encoder_test http_guard_test moe_gguf_index_test laya_bounds_test \
npu_model_bounds_test npu_forward_loader_test npu_design_grid_test npu_sampling_test \
npu_flm_head_import_test npu_elf_args_test \
npu_template_test npu_chatml_test \
tokenizer_linear_test hrx_placement_test hrx_cwsr_test hrx_runtime_test npu_device_lock_test -j
- name: Test
# One hung test must not hold the job for hours: each test gets 10 minutes.
Expand Down
62 changes: 62 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,46 @@ file(WRITE ${CMAKE_BINARY_DIR}/generated/hrx_archs.h
set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${CMAKE_SOURCE_DIR}/registry/architectures.json)
set_target_properties(onebit PROPERTIES OUTPUT_NAME 1bit)
target_link_libraries(onebit PRIVATE nlohmann_json::nlohmann_json httplib::httplib Threads::Threads)

# ── The chat-template renderer (engine #963, docs/npu-chat-template.md) ──────
# The NPU chat route renders the container's own `chat_template` (app/npu_template.h) with the pinned
# llama.cpp fork's Jinja engine (`third_party/llama.cpp/common/jinja`, upstream's, the one
# `llama-server --jinja` uses) rather than with a second, weaker interpreter of the same language.
#
# The sources are the fork's, so they exist exactly when the submodule is checked out: the option
# defaults to that, and a tree that never initialised the submodules still builds -- without the
# renderer, which the load line and the route say out loud rather than rendering something else
# (app/npu_template.h). Nothing is downloaded at build time.
if(EXISTS ${CMAKE_SOURCE_DIR}/third_party/llama.cpp/common/jinja/lexer.cpp)
set(_onebit_jinja_default ON)
else()
set(_onebit_jinja_default OFF)
endif()
option(ONEBIT_JINJA "Build the Jinja chat-template renderer from the llama.cpp submodule" ${_onebit_jinja_default})
if(ONEBIT_JINJA)
if(NOT EXISTS ${CMAKE_SOURCE_DIR}/third_party/llama.cpp/common/jinja/lexer.cpp)
message(FATAL_ERROR "ONEBIT_JINJA=ON but third_party/llama.cpp/common/jinja is missing: "
"git submodule update --init third_party/llama.cpp")
endif()
set(_onebit_jinja_dir ${CMAKE_SOURCE_DIR}/third_party/llama.cpp/common)
add_library(onebit_jinja STATIC
${_onebit_jinja_dir}/jinja/lexer.cpp
${_onebit_jinja_dir}/jinja/parser.cpp
${_onebit_jinja_dir}/jinja/runtime.cpp
${_onebit_jinja_dir}/jinja/value.cpp
${_onebit_jinja_dir}/jinja/string.cpp
${_onebit_jinja_dir}/jinja/caps.cpp
# jinja/value.cpp includes the fork's unicode.h, and these are its implementation
${_onebit_jinja_dir}/unicode.cpp)
target_include_directories(onebit_jinja PUBLIC ${_onebit_jinja_dir})
target_link_libraries(onebit_jinja PUBLIC nlohmann_json::nlohmann_json)
target_compile_definitions(onebit_jinja PUBLIC ONEBIT_JINJA=1)
target_compile_definitions(onebit PRIVATE ONEBIT_JINJA=1)
target_link_libraries(onebit PRIVATE onebit_jinja)
message(STATUS "ONEBIT_JINJA=ON — the NPU chat route renders the container's own chat template")
else()
message(STATUS "ONEBIT_JINJA=OFF — no chat-template renderer: the NPU chat route keeps ChatML (#963 option 1)")
endif()
if(ONEBIT_ORT)
add_subdirectory(onnx)
target_link_libraries(onebit PRIVATE onebit_ort_info onebit_ort_run)
Expand Down Expand Up @@ -686,6 +726,28 @@ add_test(NAME npu_chatml COMMAND npu_chatml_test ${CMAKE_SOURCE_DIR}/tests/data/
# target is not built reports "Not Run" (#635). This lane's token cannot edit the workflow.
add_dependencies(npu_sampling_test npu_chatml_test)

# The container's own chat template rendered (app/npu_template.h, engine #963 option 2): the golden rows are
# Jinja2 3.1.6's own renders of the fixtures' templates, so the assertion is a comparison against an oracle
# that is not this code. Device-free. Without the renderer (ONEBIT_JINJA=OFF) the two fixture rows cannot
# pass, so the test says so in its first line rather than skipping quietly.
add_executable(npu_template_test tests/npu_template_test.cpp)
# app/template_pin.h includes the pins compiled in from config/template-pins.json (the same read the
# load line does), which CMake writes into the build's generated/ at configure time.
target_include_directories(npu_template_test PRIVATE ${CMAKE_SOURCE_DIR}/app ${CMAKE_SOURCE_DIR}/npu
${CMAKE_BINARY_DIR}/generated)
target_link_libraries(npu_template_test PRIVATE onebit_npu nlohmann_json::nlohmann_json)
if(ONEBIT_JINJA)
target_link_libraries(npu_template_test PRIVATE onebit_jinja)
target_compile_definitions(npu_template_test PRIVATE ONEBIT_JINJA=1)
add_test(NAME npu_template
COMMAND npu_template_test ${CMAKE_SOURCE_DIR}/tests/data/templates/qwen3_flm.json
${CMAKE_SOURCE_DIR}/tests/data/templates/zr1.json
--tokenizer ${CMAKE_SOURCE_DIR}/tests/data/tokenizer_chatml.json)
else()
add_test(NAME npu_template COMMAND npu_template_test --no-renderer)
set_tests_properties(npu_template PROPERTIES SKIP_RETURN_CODE 77)
endif()

# `1bit unified`'s wait for the single NPU (app/npu_queue.h, engine#1024): bounded, and a queued request says so
# instead of leaving a stream silent past the proxy's no-data timer. Device-free.
add_executable(npu_queue_test tests/npu_queue_test.cpp)
Expand Down
72 changes: 59 additions & 13 deletions app/forward_serve.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
#include "npu_device_lock.h"
#include "npu_answer_parser.h"
#include "npu_chatml.h"
#include "npu_template.h" // the container's own chat template (#963 option 2)
#include "npu_grammar.h"
#include "npu_load_retry.h" // engine#1241: a refused NPU context names another client, not a dead device
#include "npu_prefix_cache.h"
Expand Down Expand Up @@ -87,8 +88,16 @@ struct ForwardEngine {
// The chat route renders ChatML for every container (#832), so whether this container's
// tokenizer can represent that prompt is a property of the container: decided once at load
// (#963). A refusal, not a load failure -- the id route stays usable on such a container.
bool chatml_ok = true;
std::string chatml_refusal; // empty unless !chatml_ok; the 400's message
// Since #963 option 2 the route renders the container's own template when it declares one, so
// the flag is "this container can be served on the chat route at all", not "its template is
// ChatML": a container with a template needs only that the build carries a renderer.
bool chat_ok = true;
std::string chat_refusal; // empty unless !chat_ok; the 400's message
// The template the container declares, its sha256 and where it came from: rendered per request
// when present (app/npu_template.h), else ChatML when the tokenizer has the anchors.
npu::ChatTemplate tmpl;
std::string bos_text; // the bos/eos *strings* the template's {{ bos_token }} renders
std::string eos_text;
// decode() of every vocabulary id, built on the first constrained request: the GBNF
// mask tests each token's text against the grammar state, per token.
std::vector<std::string> tok_text;
Expand Down Expand Up @@ -179,8 +188,23 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res
bool prompt_is_ids = false;
try {
if (chat) {
if (!e.chatml_ok) throw std::runtime_error(e.chatml_refusal);
prompt = npu::chatml(body.at("messages"), npu::thinking_from_body(body));
if (!e.chat_ok) throw std::runtime_error(e.chat_refusal);
if (e.tmpl.present() && npu::renderer_built()) {
// The request's own knobs reach the template as variables, the way llama-server hands
// `chat_template_kwargs` over: FastFlowLM's packs branch on `user_system_prompt`, Qwen3's
// on `enable_thinking`. Ordered re-read of the same bytes: `body` is a std::map, and a
// template that serializes its input (a tools block through `tojson`) renders the
// request's key order into the prompt, so it has to be the caller's order, not ours.
const nlohmann::ordered_json ordered = nlohmann::ordered_json::parse(req.body);
nlohmann::ordered_json extra = nlohmann::ordered_json::object();
if (ordered.contains("chat_template_kwargs") && ordered["chat_template_kwargs"].is_object())
extra = ordered["chat_template_kwargs"];
extra["enable_thinking"] = npu::thinking_from_body(body);
prompt = npu::render_chat_template(e.tmpl, ordered.at("messages"), /*add_generation_prompt=*/true,
extra, e.bos_text, e.eos_text);
} else {
prompt = npu::chatml(body.at("messages"), npu::thinking_from_body(body));
}
} else {
prompt_is_ids = npu::prompt_ids_from_body(body, e.tok->size(), prompt_ids);
if (!prompt_is_ids) prompt = body.at("prompt").get<std::string>();
Expand Down Expand Up @@ -247,8 +271,11 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res
std::vector<int> ids = prompt_is_ids ? prompt_ids : e.tok->encode(prompt);
// A chat template that opens with {{- bos_token }} (MiniCPM5 and other
// llama-class models) needs the BOS id: the ChatML prompt built above never
// carries it, and those models derail into their special tokens without it.
if (!prompt_is_ids && chat && e.add_bos && e.bos_id >= 0) ids.insert(ids.begin(), e.bos_id);
// carries it, and those models derail into their special tokens without it. A rendered
// template may already have emitted it, and a second BOS is a prompt no HF tokenizer
// produces.
if (!prompt_is_ids && chat && e.add_bos && e.bos_id >= 0 && (ids.empty() || ids.front() != e.bos_id))
ids.insert(ids.begin(), e.bos_id);
if (ids.empty()) throw std::runtime_error("empty prompt");
stats.prompt_tokens = int(ids.size());
// The KV cache grows with every position: the prompt and the answer together stay
Expand Down Expand Up @@ -810,13 +837,27 @@ int run_forward_serve(int argc, char** argv) {
e.max_context = q4nx_config_int(cfg.c_str(), "max_position_embeddings", 0);
if (e.max_context <= 0) e.max_context = 32768;

// #963: the chat route's prompt is ChatML for every container, and a tokenizer with no
// #963: the chat route's prompt was ChatML for every container, and a tokenizer with no
// <|im_start|>/<|im_end|> added token cannot represent it -- the marker becomes ordinary pieces and
// the model reads text its own template never produces (zyphra/ZR1-1.5B: 33 tokens rendered
// against 15 under the model's template). Refused per request rather than refused at load,
// because /v1/completions with token ids (#938) is the way to serve such a container correctly.
if (!npu::chatml_supported(*e.tok)) {
e.chatml_ok = false;
// against 15 under the model's template). Since option 2 the container's own template is rendered
// when it declares one, so the anchors are only needed by a container that declares none; a
// container with neither is still refused per request rather than failed at load, because
// /v1/completions with token ids (#938) serves it correctly.
e.tmpl = npu::load_chat_template(dir);
// The load names the template's identity, the same number a pin or a 400 quotes (#1089, #1095):
// one line per load, and "none" is a fact worth printing rather than an absence.
std::fprintf(stderr, "1bit forward-serve: template sha256=%s (%s)%s\n",
e.tmpl.sha256.empty() ? "none" : e.tmpl.sha256.c_str(),
e.tmpl.present() ? e.tmpl.source.c_str() : "no template",
e.tmpl.present() ? "" : " -- the chat route uses ChatML with anchors, else refuses");
const bool can_render = e.tmpl.present() && npu::renderer_built();
if (e.tmpl.present() && !npu::renderer_built())
std::fprintf(stderr,
"1bit forward-serve: this build has no chat-template renderer (ONEBIT_JINJA=OFF), so the "
"container's template is NOT used and the chat route falls back to ChatML\n");
if (!can_render && !npu::chatml_supported(*e.tok)) {
e.chat_ok = false;
std::string specials;
std::vector<std::pair<const char*, int>> named = {{"bos", e.bos_id}};
for (int id : e.eos) named.emplace_back("eos", id);
Expand All @@ -826,13 +867,18 @@ int run_forward_serve(int argc, char** argv) {
specials += std::string(specials.empty() ? "" : ", ") + label + " " + text + " (" +
std::to_string(id) + ")";
}
e.chatml_refusal =
e.chat_refusal =
"this container's tokenizer has no <|im_start|> / <|im_end|>, so the ChatML prompt the chat route "
"renders would be tokenized into pieces the model never sees (#963); send /v1/completions with "
"prompt as token ids (#938) and render the model's own template" +
(specials.empty() ? std::string() : " (model specials: " + specials + ")");
std::fprintf(stderr, "1bit forward-serve: chat route disabled: %s\n", e.chatml_refusal.c_str());
std::fprintf(stderr, "1bit forward-serve: chat route disabled: %s\n", e.chat_refusal.c_str());
}
// The template variables `{{ bos_token }}`/`{{ eos_token }}` render: the strings the tokenizer
// actually has (a null bos_token in tokenizer_config.json is "", and the engine's -1 is the same
// fact), not a second read of the JSON that could disagree with the tokenizer.
if (e.bos_id >= 0) e.bos_text = e.tok->decode(e.bos_id);
if (!e.eos.empty() && e.eos.front() >= 0) e.eos_text = e.tok->decode(e.eos.front());

// The prompt cache (#650): --prefix-slots 0 turns it off, and an invalid value is refused here
// rather than clamped, like every other option on this command.
Expand Down
Loading
Loading