diff --git a/CMakeLists.txt b/CMakeLists.txt index d2361d2a5..287bb9ef8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -200,6 +200,12 @@ add_test(NAME serve_hangup COMMAND ${CMAKE_SOURCE_DIR}/tests/serve_hangup_test.s # engine#1945: scripts/bump-hrx-sync.sh tells AMD's own commit from a line AMD rebased apart from a commit that # is not AMD's. Local git repositories only: no network, no GPU. add_test(NAME bump_hrx_sync COMMAND bash ${CMAKE_SOURCE_DIR}/tests/bump_hrx_sync_test.sh ${CMAKE_SOURCE_DIR}/scripts/bump-hrx-sync.sh) +# engine#2102: the sentence a serve owes an operator when no 16-row decode kernel was found (app/npu_decode_note.h). Pure: +# the silent cases are the ones that matter, and they are arms. +add_executable(npu_decode_note_test tests/npu_decode_note_test.cpp) +target_include_directories(npu_decode_note_test PRIVATE app) +add_test(NAME npu_decode_note COMMAND npu_decode_note_test) +add_dependencies(onebit npu_decode_note_test) add_test(NAME bump_hrx_report COMMAND bash ${CMAKE_SOURCE_DIR}/tests/bump_hrx_report_test.sh ${CMAKE_SOURCE_DIR}/scripts/bump-hrx-report.sh) add_test(NAME metrics_serve COMMAND ${CMAKE_SOURCE_DIR}/tests/metrics_serve.sh $) # serve --print-backend: the llama-server Lemonade starts itself, one hop (docs/serve.md): no GPU. @@ -306,6 +312,11 @@ add_python_test(python_test_order_check ${CMAKE_SOURCE_DIR}/tests/python_test_or add_python_test(python_test_order_check_self ${CMAKE_SOURCE_DIR}/tests/python_test_order_check.py --self-test) add_python_test(ci_targets ${CMAKE_SOURCE_DIR}/tools/ci/ci_targets_check.py --check --root ${CMAKE_SOURCE_DIR}) add_python_test(ci_targets_self ${CMAKE_SOURCE_DIR}/tools/ci/ci_targets_check.py --self-test) +# engine#803 gate, second half: the comparator that turns two --dump-logits directories into the KLD verdict the +# P4 gate asks for. No device and no model, so the gate is one command when a ticket is free; the self-test pins +# the four cases a KLD implementation gets wrong (identical -> exactly 0, a shift is positive and directional, a +# uniform shift stays 0, and an asymmetric file set is refused by name). +add_python_test(logits_kld_self ${CMAKE_SOURCE_DIR}/tools/logits_kld.py --self-test) # engine#2018: this one was in tools/ci/unreferenced_py_tests.txt as a baseline entry, and it should not have been: the # file is portable and passes (worst-case |sum| 11.18e6 inside int32 at K=11008, 4 arms), it just never had a runner. A # baseline records a test as never-run, which for a passing test is the wrong outcome -- so it is registered and the line @@ -323,6 +334,13 @@ add_python_test(tests_referenced_self ${CMAKE_SOURCE_DIR}/tools/ci/tests_referen # Registered through add_python_test (engine#1864), so they run THROUGH the interpreter: my first version used the raw # `${Python3_EXECUTABLE}` form above the find_package and the repository's own ordering guard caught it by name -- # tools/mtp_head_ref.py is the one of the five without an exec bit, and it became BAD_COMMAND under ctest. +# engine#1962: the narrowing-body enumeration, as a ratchet. The corpus is a submodule, so the check +# skips (rc 77) where third_party/llama.cpp is not checked out; the baseline file is in the engine tree, +# so the recorded set is reviewable everywhere. +add_python_test(tool_selftest_hrx_f16_narrowing ${CMAKE_SOURCE_DIR}/tools/ci/hrx_f16_narrowing.py --self-test) +add_test(NAME hrx_f16_narrowing_check COMMAND ${Python3_EXECUTABLE} + ${CMAKE_SOURCE_DIR}/tools/ci/hrx_f16_narrowing.py --check --root ${CMAKE_SOURCE_DIR}) +set_tests_properties(hrx_f16_narrowing_check PROPERTIES SKIP_RETURN_CODE 77) add_python_test(tool_selftest_batched_repeat ${CMAKE_SOURCE_DIR}/tools/batched_repeat.py --selftest) add_python_test(tool_selftest_logprob_agree ${CMAKE_SOURCE_DIR}/tools/logprob_agree.py --selftest) add_python_test(tool_selftest_mtp_head_ref ${CMAKE_SOURCE_DIR}/tools/mtp_head_ref.py --selftest) @@ -358,7 +376,19 @@ add_test(NAME bench_preflight COMMAND bash ${CMAKE_SOURCE_DIR}/tools/tests/test- # two false conclusions about a pack in one night. `--self-test` drives the rule on a synthetic shard (no network): # the same offset from the data section returns the tensor, from byte 8 it does not. add_python_test(hf_tensor_self ${CMAKE_SOURCE_DIR}/tools/hf_tensor.py --self-test) +# engine#1946: a test that returns 77 is green, and green-because-it-did-not-run is the failure this fleet keeps paying +# for. `tools/ci/ctest_skips.py` reads a ctest log and reports every test that did not execute -- `***Skipped` (77) against +# `tools/ci/ctest_skips.txt`, and `(Not Run)` (a missing binary) as a build-list mistake that can never be baselined. +# Self-test only here: wiring it into ci.yml's Test step needs the log plus a `workflow`-scoped token (engine#1817). +add_python_test(ctest_skips_self ${CMAKE_SOURCE_DIR}/tools/ci/ctest_skips.py --self-test) +# engine#2106 / #2102: the release gate's NPU serve check asserts the fast route, a decode floor, the first token and a +# non-empty answer, because a serve on the slow per-op route still answers. --self-test drives the judging on the shape +# of each failure (the silent fallback at 1.9 tok/s, thinking left on, a degenerate reply); no device, no binary. +add_python_test(npu_serve_smoke_self ${CMAKE_SOURCE_DIR}/tools/npu_serve_smoke.py --self-test) add_python_test(moe_placement ${CMAKE_SOURCE_DIR}/tools/moe_placement.py --self-test) +# engine#1313 item 3: the "after" of "before/after routing accuracy" -- a ridge head over --dump-encoder output, with the +# table's own logits as the before arm. Self-test only, so it needs no dump and no model. +add_python_test(laya_head_finetune ${CMAKE_SOURCE_DIR}/tools/laya_head_finetune.py --self-test) # engine#856: one HRX process's GTT split into the loader's copy and HRX's packed second copy, from its own -v log. add_python_test(hrx_gtt_split_self ${CMAKE_SOURCE_DIR}/tools/hrx_gtt_split.py --self-test) # engine#679: time per HRX dispatch from the backend's serial trace (the id hash and the per-kernel sums). @@ -373,7 +403,18 @@ add_test(NAME pack_version COMMAND ${CMAKE_SOURCE_DIR}/tests/pack_version_test.s # resolves outside the payload, and a runtime set shipped without the aqlprofile agent libhsa opens. Each # arm asserts the row's message, and a clean tree is the control. No package build, no GPU, runs in CI. add_test(NAME pack_check COMMAND ${CMAKE_SOURCE_DIR}/tests/pack_check_test.sh) +# engine#2106: the hardware gate's `npuserve` phase, end to end against the stand-in serve tools/npu_serve_smoke.py +# writes: a fast-route serve passes the gate, a slow-route one fails it while its answers are fine, a dead one fails, +# and an unconfigured or NPU-less host skips. No device, no 1bit binary, and the fleet's queues are not touched. +add_test(NAME hw_gate_npuserve COMMAND ${CMAKE_SOURCE_DIR}/tests/hw_gate_npuserve_test.sh) +# Both start a loopback serve on a port picked by binding port 0 and releasing it, so they take the lock every +# loopback-server test takes (tests/route_port_lock_test.py): two of them racing for one freed port is a flaky failure. +set_tests_properties(hw_gate_npuserve npu_serve_smoke_self PROPERTIES RESOURCE_LOCK route_ports) add_test(NAME check_ci_runners COMMAND ${CMAKE_SOURCE_DIR}/tests/check_ci_runners_test.sh) +# engine #1504: the release preflight blocks on a runner unit that is down and only warns on a short restart policy. +# The real scripts/ci-runner-health.sh and weekly.sh's preflight_runner_units() against a stub systemctl; no systemd. +add_test(NAME weekly_runner_preflight COMMAND ${CMAKE_SOURCE_DIR}/tests/weekly_runner_preflight_test.sh) +add_test(NAME ci_runner_health_selftest COMMAND bash ${CMAKE_SOURCE_DIR}/scripts/ci-runner-health.sh --self-test) # engine #1750: the carried amdxdna series applies in order to the pinned driver, and the context-restart code it # produces fails a context's jobs (and logs it) when the restart fails, instead of leaving them parked or running # them at close on a half-built firmware context. The driver's own functions in a user-space harness: no NPU, no @@ -823,6 +864,12 @@ add_executable(laya_eval_test tests/laya_eval_test.cpp) target_link_libraries(laya_eval_test PRIVATE onebit_laya nlohmann_json::nlohmann_json) add_test(NAME laya_eval COMMAND laya_eval_test) add_dependencies(onebit laya_eval_test) +# engine#1313 item 3: the head-only fine-tune over --dump-encoder dumps, on a synthetic checkpoint and dump (no model, +# no device). It needs torch + numpy, which the engine build does not, so a box without them skips (77). +add_python_test(laya_finetune ${CMAKE_SOURCE_DIR}/tests/laya_finetune_test.py) +if(Python3_Interpreter_FOUND) + set_tests_properties(laya_finetune PROPERTIES SKIP_RETURN_CODE 77) +endif() # Crafted Laya checkpoints the reader and the scorer must refuse (written at run time); runs in CI. add_executable(laya_bounds_test tests/laya_bounds_test.cpp) target_link_libraries(laya_bounds_test PRIVATE onebit_laya nlohmann_json::nlohmann_json) @@ -894,6 +941,12 @@ if(Python3_Interpreter_FOUND) COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/ci/npu_decl_check.py --check --root ${CMAKE_SOURCE_DIR}) add_test(NAME npu_decl_check_self COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/ci/npu_decl_check.py --self-test) + # engine#2057: no fopen(..., "w"/"a") in the tree without a stated reason -- CodeQL's world-writable class, kept fixed + # by a check rather than by the next scan. Source text only: no build, no device. + add_test(NAME file_create_check + COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/ci/file_create_check.py --check --root ${CMAKE_SOURCE_DIR}) + add_test(NAME file_create_check_self + COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/ci/file_create_check.py --self-test) # the new-model watcher's alert contract (offline: synthetic batches against the # committed registry, plus FastFlowLM's org listing and releases from the recorded # payloads under tests/fixtures/fflm/ — engine #588), so a regression cannot leave the @@ -1293,10 +1346,25 @@ target_include_directories(npu_pack_test PRIVATE npu/forward/include ${CMAKE_SOU # NPU_INFER_DUMP_OPS (npu/forward/include/ops-dump.h, engine#1503): the on/off rule, the mean|x|/absmax arithmetic and # the line's text. Header-only, no device. `add_dependencies(onebit ...)` is what gets it built in CI, whose target # list is explicit (tools/ci/ci_targets_check.py). +# npu/forward/include/file_create.h (engine#2057): the openers every dump and the head-image cache write through -- 0600 / +# at most 0644 whatever the umask, an existing 0666 file narrowed, a symlink refused with ELOOP. Plain C, no model, no +# device; the call sites only swapped fopen() for an opener, so without this arm a revert to fopen() passes every test. +if(NOT WIN32) + add_executable(file_create_test tests/file_create_test.c) + target_include_directories(file_create_test PRIVATE npu/forward/include) + add_test(NAME file_create COMMAND file_create_test) + add_dependencies(onebit file_create_test) +endif() add_executable(ops_dump_test tests/ops_dump_test.cpp) target_include_directories(ops_dump_test PRIVATE npu/forward/include) add_test(NAME ops_dump COMMAND ops_dump_test) add_dependencies(onebit ops_dump_test) +# When the dense-tail cache is on by default (npu/forward/include/ffn_cache_policy.h): the size rule and the two +# models it was measured on, one on each side of the old floor. Header-only, no device. +add_executable(ffn_cache_policy_test tests/ffn_cache_policy_test.cpp) +target_include_directories(ffn_cache_policy_test PRIVATE npu/forward/include) +add_test(NAME ffn_cache_policy COMMAND ffn_cache_policy_test) +add_dependencies(onebit ffn_cache_policy_test) # The device weight layout contract (q4nx_weight_layout): pure index math, no device # and no runtime. CI builds an explicit target list, so the check compiles itself with @@ -1736,6 +1804,12 @@ if(ONEBIT_NPU) target_include_directories(hybrid_prompt_probe PRIVATE ${CMAKE_SOURCE_DIR}/npu ${CMAKE_SOURCE_DIR}/npu/forward/include ${CMAKE_SOURCE_DIR}/npu/forward/engine-src) target_link_libraries(hybrid_prompt_probe PRIVATE onebit_npu_forward onebit_npu_rt) + # engine#1788: one picture and one question through a MiniCPM-V 4.7 pack with every step the engine's own + # (plan, resize, host tower, prompt tokens, canvas positions, then step_batch_embeds on the text model). + add_executable(minicpm_image_probe npu/forward/minicpm_image_probe.cpp) + target_include_directories(minicpm_image_probe PRIVATE ${CMAKE_SOURCE_DIR}/npu + ${CMAKE_SOURCE_DIR}/npu/forward/include ${CMAKE_SOURCE_DIR}/npu/forward/engine-src) + target_link_libraries(minicpm_image_probe PRIVATE onebit_npu onebit_npu_forward onebit_npu_rt) add_executable(mrope_planes_probe npu/forward/mrope_planes_probe.cpp) target_include_directories(mrope_planes_probe PRIVATE ${CMAKE_SOURCE_DIR}/npu ${CMAKE_SOURCE_DIR}/npu/forward/include ${CMAKE_SOURCE_DIR}/npu/forward/engine-src) diff --git a/app/forward_serve.cpp b/app/forward_serve.cpp index 1b4249bec..db836fc45 100644 --- a/app/forward_serve.cpp +++ b/app/forward_serve.cpp @@ -35,6 +35,9 @@ // resets it from position 0. #include "forward_serve.h" +#include "../npu/forward/include/file_create.h" // dumps are created 0600, never through a symlink +#include "npu_decode_note.h" + #include "../npu/forward/include/q4nx_forward.h" #include "../npu/target/xdna/xdna_target.h" // XdnaTarget::kName for --npu-target #include "npu_device_lock.h" @@ -153,6 +156,10 @@ struct CallStats { double prompt_ms = 0; double predicted_ms = 0; npu::PrefixDecision prefix{-1, 0, "off"}; // "off" when --prefix-slots 0 + // engine#2111: each generated token's text, in order, and how many of them were reasoning. -1 = not counted (a + // /v1/completions reply is raw text with no split), so the usage object then carries no details at all. + std::vector pieces; + int reasoning_tokens = -1; }; // One generated token's log-probability entry, with the text that token produced. The two travel @@ -530,7 +537,7 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib if (!e.dump_logits.empty()) { const std::string path = e.dump_logits + "/step_" + std::to_string(dump_req) + "_" + std::to_string(n) + ".bin"; - if (std::FILE* f = std::fopen(path.c_str(), "wb")) { + if (std::FILE* f = onebit_fopen_private_wb(path.c_str())) { // 0600, no symlink (file_create.h) std::fwrite(logits.data(), sizeof(float), logits.size(), f); std::fclose(f); } @@ -549,6 +556,7 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib // pre-top_k/top_p/temperature and the grammar mask cannot narrow it. if (want_lp) held.emplace_back(Emitted{piece, npu::logprobs_of(logits, next, opts.sample.top_logprobs)}, piece.size()); + stats.pieces.push_back(piece); // engine#2111: reasoning_tokens is counted over these ++produced; const size_t stop_at = npu::find_stop(pending, opts.stop); if (stop_at != std::string::npos) { @@ -597,14 +605,19 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib // OpenAI's usage object plus the two fields issue #650 reports: how many prompt tokens came // from a saved prefix, and what the cache decided (which slot, and why). auto usage_json = [](const CallStats& st) { - return json{{"prompt_tokens", st.prompt_tokens}, - {"completion_tokens", st.completion_tokens}, - {"total_tokens", st.prompt_tokens + st.completion_tokens}, - {"prompt_tokens_details", {{"cached_tokens", st.prefix.cached_tokens}}}, - {"flm_prefix_cache", {{"slot", st.prefix.slot}, - {"cached_tokens", st.prefix.cached_tokens}, - {"reason", st.prefix.reason}, - {"pinned", st.prefix.pinned}}}}; + json u{{"prompt_tokens", st.prompt_tokens}, + {"completion_tokens", st.completion_tokens}, + {"total_tokens", st.prompt_tokens + st.completion_tokens}, + {"prompt_tokens_details", {{"cached_tokens", st.prefix.cached_tokens}}}, + {"flm_prefix_cache", {{"slot", st.prefix.slot}, + {"cached_tokens", st.prefix.cached_tokens}, + {"reason", st.prefix.reason}, + {"pinned", st.prefix.pinned}}}}; + // engine#2111: OpenAI's field for "how much of the budget went into reasoning". With content "" and + // finish_reason "length", reasoning_tokens == completion_tokens is the sentence that says the budget ran out + // inside a thinking block, rather than that the model said nothing. Additive: absent where nothing was split. + if (st.reasoning_tokens >= 0) u["completion_tokens_details"] = {{"reasoning_tokens", st.reasoning_tokens}}; + return u; }; // OpenAI's logprobs.content entry: the token's text, its log-probability, and the most likely @@ -639,6 +652,7 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib // parts; a reply that is only tool calls says content: null. const npu::ParsedAnswer ans = npu::parse_answer(content, family, opts.tools_sent, starts_in_reasoning); + stats.reasoning_tokens = npu::reasoning_token_count(stats.pieces, family, opts.tools_sent, starts_in_reasoning); json message{{"role", "assistant"}}; message["content"] = ans.content_null ? json(nullptr) : json(ans.content); if (!ans.reasoning.empty()) message["reasoning_content"] = ans.reasoning; @@ -767,6 +781,10 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib { json tail = chunk_tpl; tail["choices"] = json::array(); + // engine#2111: the same count as the non-streamed reply's, over the same pieces. + if (chat) + stats.reasoning_tokens = + npu::reasoning_token_count(stats.pieces, family, opts.tools_sent, starts_in_reasoning); tail["usage"] = usage_json(stats); tail["timings"] = timings_json(stats); send("data: " + tail.dump() + "\n\n"); @@ -807,6 +825,12 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib // this thread waits for the first frame or the end, and only then decides between an ordinary response -- the 400s // and 500s a request can earn before it has produced anything keep their status codes -- and a stream. void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Response& res, bool chat) { + // A non-causal decision pack has no decode to run (npu::decision_only_refusal): say so before any work. + if (e.fwd && e.fwd->noncausal()) { + res.status = 400; + res.set_content(npu::decision_only_refusal(e.id, e.head != nullptr).dump(), "application/json"); + return; + } bool wants_stream = false; try { const json body = json::parse(req.body); @@ -1152,9 +1176,27 @@ int run_forward_serve(int argc, char** argv) { // after the template, because whether the route renders ChatML is known only now. { const bool chatml_route = !can_render && e.chat_ok; + // The marker this route's prompts close an assistant turn with. On the ChatML fallback it is <|im_end|>. + // With the container's own template it is read off the template: render a two-message exchange and take + // what follows the assistant's text (npu::turn_end_marker). A template that cannot render that exchange + // contributes nothing, and the load line below shows what the serve stops on either way. + std::string turn_end = chatml_route ? std::string("<|im_end|>") : std::string(); + if (can_render) { + static const char* const kProbe = "\xC2\xA4turn-end-probe\xC2\xA4"; + try { + nlohmann::ordered_json exchange = nlohmann::ordered_json::array(); + exchange.push_back({{"role", "user"}, {"content", "u"}}); + exchange.push_back({{"role", "assistant"}, {"content", kProbe}}); + const std::string rendered = npu::render_chat_template(e.tmpl, exchange, /*add_generation_prompt=*/false, + nlohmann::ordered_json::object(), e.bos_text, e.eos_text); + const size_t at = rendered.find(kProbe); + if (at != std::string::npos) turn_end = npu::turn_end_marker(rendered.substr(at + std::strlen(kProbe))); + } catch (const std::exception& ex) { + std::fprintf(stderr, "1bit forward-serve: the template did not render a two-message exchange (%s); no turn-end marker from it\n", ex.what()); + } + } e.eos = npu::stop_token_ids(npu::read_json_or_null(cfg), npu::read_json_or_null(dir + "/tokenizer_config.json"), - chatml_route ? std::string("<|im_end|>") : std::string(), - [&](const std::string& text) { return e.tok->token_id(text); }); + turn_end, [&](const std::string& text) { return e.tok->token_id(text); }); std::string named; for (size_t i = 0; i < e.eos.size(); i++) named += std::string(i ? ", " : "") + std::to_string(e.eos[i]) + " " + e.tok->decode(e.eos[i]) + @@ -1359,6 +1401,13 @@ int run_forward_serve(int argc, char** argv) { // The target that is actually serving, so a log says which one answered rather than leaving it to the build. std::fprintf(stderr, " [npu-target=%s]", npu_target.c_str()); std::fprintf(stderr, "\n"); + // engine#2102: the machine line `[forward] decode family: …` stays byte-for-byte (gates grep it). This is the sentence + // an operator needs at the ready line's level when it says 0 of N -- the route that answers correctly and 50x slower. + { + const std::string note = onebit::npu::decode_family_note(e.fwd->decode_roles(), e.fwd->design_roles(), kernels, + e.fwd->decode_disabled(), e.fwd->decode_image()); + if (!note.empty()) std::fprintf(stderr, "%s\n", note.c_str()); + } while (true) std::this_thread::sleep_for(std::chrono::seconds(3600)); return 0; diff --git a/app/npu_answer_parser.h b/app/npu_answer_parser.h index ab1c95ee2..f6453fcb0 100644 --- a/app/npu_answer_parser.h +++ b/app/npu_answer_parser.h @@ -172,6 +172,9 @@ class AnswerStream { // The calls seen so far, in order. A call needs tools_sent, per the issue's rule. const std::vector& calls() const { return calls_; } bool saw_reasoning() const { return saw_reasoning_; } + // Inside a reasoning block right now (after the opening tag, before the closing one). reasoning_token_count() + // reads it before and after each token, so the tags themselves and held-back partial tags count with the block. + bool in_reasoning() const { return mode_ == Mode::reasoning; } bool saw_text() const { return saw_text_; } const std::string& reasoning() const { return reasoning_; } @@ -180,8 +183,22 @@ class AnswerStream { private: enum class Mode { content, reasoning, call }; - void emit(std::vector>& out, bool is_reasoning, const std::string& text) { - if (text.empty()) return; + // Whitespace around a reasoning block is the model's layout, not part of either text: `\n...\n\n\n` + // then the answer. Left in, the answer to "one word" comes back as "\n\nParis", and a model that thinks nothing + // (`\n\n`) sends a reasoning chunk that is two newlines. So the whitespace that LEADS the reasoning + // is dropped, and so is the whitespace that leads the content when that content follows a reasoning block -- + // the rule app/think_split.h has always applied on the in-process route ("whitespace around the block is + // dropped"), so the two routes return the same strings. Content with no reasoning before it is not touched. + void emit(std::vector>& out, bool is_reasoning, const std::string& raw) { + if (raw.empty()) return; + std::string text = raw; + bool& trim = is_reasoning ? trim_reasoning_ : trim_content_; + if (trim) { + const size_t b = text.find_first_not_of(" \t\r\n"); + if (b == std::string::npos) return; // all whitespace so far: nothing to send yet + text.erase(0, b); + trim = false; + } if (is_reasoning) { reasoning_ += text; saw_reasoning_ = true; @@ -221,6 +238,8 @@ class AnswerStream { } if (starts(tags_.think_close)) { flush_plain(); + // what follows a block is separated from it by layout whitespace -- unless content came first + if (mode_ == Mode::reasoning && !saw_text_) trim_content_ = true; mode_ = Mode::content; i += std::string(tags_.think_close).size(); continue; @@ -377,6 +396,8 @@ class AnswerStream { std::vector calls_; bool saw_reasoning_ = false; bool saw_text_ = false; + bool trim_reasoning_ = true; // until the reasoning's first non-whitespace character + bool trim_content_ = false; // set when a reasoning block closes before any content; cleared by the first text }; // Does the rendered prompt leave the model INSIDE a reasoning block? Some templates end their generation prompt @@ -424,6 +445,57 @@ inline std::vector content_mask(const std::vector& pieces, An return mask; } +// How many of `pieces` (one generated token's text each, in order) were reasoning: OpenAI's +// usage.completion_tokens_details.reasoning_tokens (engine#2111). An empty `content` with finish_reason "length" reads +// like "the model said nothing" when the whole budget went into a thinking block, and this is the number that says so. +// A token is reasoning when none of its text reached `content` and the block was open before it, after it, or it put +// text into the block: so the opening and closing tags, a partial tag the holdback keeps, and an unclosed block at the +// end of the budget all count, while call markup and the whitespace after a closed block do not. A token that +// straddles the boundary ("The") is content, as content_mask() says. Same splitter, so the count, the split +// and the logprob entries cannot disagree. +inline int reasoning_token_count(const std::vector& pieces, AnswerFamily family, bool tools_sent, + bool starts_in_reasoning = false) { + AnswerStream s(family, tools_sent, starts_in_reasoning); + std::vector> out; + int n = 0; + bool last_counted = false; + // Tokens that produced nothing before anything was produced (leading whitespace, the first half of an opening tag + // the holdback keeps) belong to whatever comes next: to the block if the next token opens it, to nothing otherwise. + int pending = 0; + bool started = false; + for (const auto& piece : pieces) { + const bool before = s.in_reasoning(); + out.clear(); + s.feed(piece, out); + bool content = false, reasoning = false; + for (const auto& [is_reasoning, text] : out) { + (void)text; + (is_reasoning ? reasoning : content) = true; + } + last_counted = !content && (reasoning || before || s.in_reasoning()); + if (last_counted) { + ++n; + if (!started) n += pending; + started = true; + } else if (content) { + started = true; + } else if (!started) { + ++pending; + } + } + // What the holdback still holds is flushed by finish(): if it is content, the last token's text reached `content`. + out.clear(); + s.finish(out); + for (const auto& [is_reasoning, text] : out) { + (void)text; + if (!is_reasoning && last_counted) { + --n; + last_counted = false; + } + } + return n; +} + // The whole answer at once: the same splitter, so the two paths cannot disagree. inline ParsedAnswer parse_answer(const std::string& text, AnswerFamily family, bool tools_sent, bool starts_in_reasoning = false) { diff --git a/app/npu_decode_note.h b/app/npu_decode_note.h new file mode 100644 index 000000000..0f208661c --- /dev/null +++ b/app/npu_decode_note.h @@ -0,0 +1,64 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// The sentence a serve owes an operator when it is about to run the slow route (engine#2102). +// +// The fact already exists as a machine line — `[forward] decode family: 0 role(s) at M=16, 4 on M=128` — and that line is +// grepped byte-for-byte by the gates, so it must not be reworded. What was missing is a sentence at the ready line's level: +// a serve with no 16-row decode kernels runs every decode step as prefill-width launches, measured at 1.9 tok/s where the +// fused route on the same box does 88-101, and nothing said so. This header is the whole decision, as a pure function, so it +// can be armed without a device and so the "say nothing" case is as testable as the "say something" one. +// +// It deliberately returns "" for the cases it cannot judge (a host-GEMM arm has no family at all; a count of 0 of 0 is a +// different fault, reported elsewhere) — a warning that fires on healthy serves is a warning nobody reads. + +#pragma once + +#include +#include + +namespace onebit::npu { + +// decode_roles: roles that got an M=16 decode kernel. design_roles: roles the model has (0 on the host-GEMM arm, which loads +// no decode family). kernels_dir: what the operator pointed at, echoed so the message can be acted on without reading the +// rest of the log. disabled: NPU_DECODE_M16=0 turned the family off, so nothing is missing. image: a ternary (pb16i/t2s) +// container, whose loader refuses a missing decode kernel; the roles it leaves on M=128 are ones its geometry or the split +// designs cover (named per role in the load log), so for it only 0 of N is this issue. +// +// The way out names the fused route, not the _m16 designs: measured on Qwen3-0.6B (#2102), the per-op forward with all four +// twins decodes at ~2.7 tok/s against ~1.7 without them and 88-101 on the fused route, so the twins are not the demo rate. +inline std::string decode_family_note(int decode_roles, int design_roles, const std::string& kernels_dir, + bool disabled = false, bool image = false) { + if (design_roles <= 0) return {}; // no roles (or the host arm): not this issue's fault, and no number to quote + const char* dir = kernels_dir.empty() ? "(no kernel directory)" : kernels_dir.c_str(); + char buf[640]; + if (disabled) { + std::snprintf(buf, sizeof buf, + "npu: NPU_DECODE_M16=0 turned the 16-row decode family off -- decode runs prefill-width launches on all %d " + "role(s) (engine#2102). Unset it to use the decode kernels in %s.", + design_roles, dir); + return std::string(buf); + } + if (decode_roles >= design_roles) return {}; // a full decode family: say nothing + if (image && decode_roles > 0) return {}; // a ternary family's M=128 roles are its geometry's, already named + std::snprintf(buf, sizeof buf, + "npu: %d of %d role(s) have a 16-row decode kernel in %s -- decode runs prefill-width launches and is far " + "slower than the fused route (engine#2102). The _m16 designs help only part of the way; the demo rate is the " + "fused route's (docs/npu.md, plan.json: decode.dirs).", + decode_roles, design_roles, dir); + return std::string(buf); +} + +} // namespace onebit::npu diff --git a/app/npu_stop_tokens.h b/app/npu_stop_tokens.h index d3fe3c3f7..281adf045 100644 --- a/app/npu_stop_tokens.h +++ b/app/npu_stop_tokens.h @@ -66,6 +66,34 @@ inline std::string eos_token_text(const nlohmann::json& tokenizer_config) { return {}; } +/// The marker a chat template closes an assistant turn with, read off the template's own output. `after` is the +/// text a template rendered immediately after an assistant message's content; the marker is the `<...>` token it +/// starts with (leading whitespace skipped), or empty when it starts with anything else. +/// +/// Why this exists: a model ends its turn with whatever its template ends an assistant turn with, and neither +/// config.json nor tokenizer_config.json has to name that token. FastFlowLM's Phi4-mini-Instruct-NPU2 names +/// <|endoftext|> in both, and its template closes every turn with <|end|> (200020); the upstream checkpoint +/// lists it in generation_config.json, which the pack does not ship. Served with only the two files' ids, the +/// answer was "Paris<|end|><|assistant|>**Follow-up Question 1:** ..." until the token budget ran out +/// (engine#1786). The template is the one file every chat pack has, and it cannot disagree with itself. +/// +/// "<|end|>" -> "<|end|>" (Phi) +/// "<|im_end|>\n" -> "<|im_end|>" (ChatML) +/// "<|eot_id|>" -> "<|eot_id|>" (Llama 3) +/// "\n" -> "" (Gemma) +/// "\n", "", "x"... -> "" unless it starts with a whole <...> token: plain text after a turn is not a stop. +inline std::string turn_end_marker(const std::string& after) { + size_t i = 0; + while (i < after.size() && (after[i] == ' ' || after[i] == '\n' || after[i] == '\r' || after[i] == '\t')) ++i; + if (i >= after.size() || after[i] != '<') return {}; + const size_t close = after.find('>', i); + if (close == std::string::npos || close - i < 2) return {}; + const std::string marker = after.substr(i, close - i + 1); + for (char c : marker) + if (c == ' ' || c == '\n' || c == '\r' || c == '\t') return {}; + return marker; +} + /// Every id that ends a generation on a container, in the order they were found and without repeats: /// /// 1. config.json's `eos_token_id` (one integer or a list): what eos_token_ids() reads; @@ -75,9 +103,10 @@ inline std::string eos_token_text(const nlohmann::json& tokenizer_config) { /// model ends a chat turn with the second. With only the first, the answer came back as /// "Item 3 is the 3D printer.<|im_end|>\n": the turn end served as text and one more token generated /// after it (engine#1785's row on that container); -/// 3. `chat_turn_end`, when the caller's chat route renders a prompt whose turns end with a marker of its -/// own (ChatML's <|im_end|>): a prompt we close with that marker is answered by a turn that ends with it, -/// whatever the two files say. Pass an empty string when the route has no such marker. +/// 3. `chat_turn_end`, the marker the chat route's prompt closes an assistant turn with: ChatML's <|im_end|> +/// on the fallback route, or turn_end_marker() of the container's own template when that is rendered. A +/// prompt whose turns end with that marker is answered by a turn that ends with it, whatever the two files +/// say. Pass an empty string when the route has no such marker. /// /// `id_of` maps a token's text to its id, or a negative number when the tokenizer has no such token; a name it /// does not know adds nothing. diff --git a/app/npu_systemone.h b/app/npu_systemone.h index 22623f7cc..2a6c0bfd4 100644 --- a/app/npu_systemone.h +++ b/app/npu_systemone.h @@ -124,6 +124,22 @@ inline nlohmann::json systemone_question_error(const std::string& kind, const st {"message", message}}; } +// What a text route answers on a pack that cannot generate text. A decision pack that declares +// `attention_mode: noncausal_full_attention` encodes a whole prompt in one pass and reads an answer off the last +// position; there is no token-by-token decode to run, so `/v1/chat/completions` and `/v1/completions` can never +// succeed on it. They used to try, and failed inside the forward with HTTP 500 and the forward's own wording +// ("forward step failed: attention.mode noncausal_full_attention encodes a prompt in one batched pass, but this +// call starts at position 23 ..."; measured on FastFlowLM's D3-Nano-NPU2, engine#1786). That is the client's +// request against the wrong route, so it is a 400 that names the right one. `has_head` says whether the decision +// route is actually up on this serve, because pointing a client at a route that answers 503 is no help. +inline nlohmann::json decision_only_refusal(const std::string& model, bool has_head) { + const std::string where = has_head ? "send decisions to POST /v1/systemone" + : "and this serve has no decision head loaded, so it has no route for it"; + return {{"error", + {{"message", model + " is a decision model (non-causal attention): it does not generate text; " + where}, + {"type", "decision_model_error"}}}}; +} + inline std::string systemone_length_message(size_t tokens, size_t limit) { return "the question prompt has " + std::to_string(tokens) + " tokens, over the maximum context length of " + std::to_string(limit) + " tokens; nothing was truncated"; diff --git a/app/think_split.h b/app/think_split.h index 5ba670400..efb06201f 100644 --- a/app/think_split.h +++ b/app/think_split.h @@ -16,6 +16,7 @@ #include #include +#include namespace onebit { @@ -73,4 +74,35 @@ struct ThinkSplit { } }; +// How many of `pieces` (one generated token's text each, in order) were reasoning, for usage.completion_tokens_details +// .reasoning_tokens (engine#2111): the twin of npu::reasoning_token_count() over this route's splitter. A token is +// reasoning when none of its text reached content and the block was open before it, after it, or it put text into +// it -- so both tags, a held-back partial tag and an unclosed block at the end of the budget count. +inline int think_reasoning_tokens(const std::vector& pieces, bool opened) { + ThinkSplit s(opened); + int n = 0; + bool last_counted = false; + int pending = 0; // leading tokens that produced nothing yet: they join the block if the next one opens it + bool started = false; + for (const auto& piece : pieces) { + const bool before = s.in_think; + bool content = false, reasoning = false; + s.feed(piece, [&](bool r, const std::string&) { (r ? reasoning : content) = true; }); + last_counted = !content && (reasoning || before || s.in_think); + if (last_counted) { + ++n; + if (!started) n += pending; + started = true; + } else if (content) { + started = true; + } else if (!started) { + ++pending; + } + } + bool flushed_content = false; + s.flush([&](bool r, const std::string&) { if (!r) flushed_content = true; }); + if (flushed_content && last_counted) --n; + return n; +} + } // namespace onebit diff --git a/app/unified.cpp b/app/unified.cpp index 102ab340d..c44f2a436 100644 --- a/app/unified.cpp +++ b/app/unified.cpp @@ -237,12 +237,27 @@ void handle_generate(Engine& e, const httplib::Request& req, httplib::Response& return "data: " + c.dump() + "\n\n"; }; - auto usage = [](const npu::GenerateResult& r, size_t n_prompt, size_t n_cached) { + // engine#2111: a chat reply also says how many completion tokens were reasoning (OpenAI's + // completion_tokens_details.reasoning_tokens), so content "" with finish_reason "length" can be told apart from a + // model that said nothing: the budget ran out inside the thinking block. Counted over the generated ids (the end + // token excluded, as completion_tokens excludes it) with this route's own splitter; 0 when nothing is split. + auto usage = [&e, chat, split_reasoning, think_opened](const npu::GenerateResult& r, size_t n_prompt, size_t n_cached) { const size_t completion = r.tokens.size() - (r.stopped_at_eos ? 1 : 0); - return json{{"prompt_tokens", n_prompt}, - {"completion_tokens", completion}, - {"total_tokens", n_prompt + completion}, - {"prompt_tokens_details", {{"cached_tokens", n_cached}}}}; + json u{{"prompt_tokens", n_prompt}, + {"completion_tokens", completion}, + {"total_tokens", n_prompt + completion}, + {"prompt_tokens_details", {{"cached_tokens", n_cached}}}}; + if (chat) { + int reasoning = 0; + if (split_reasoning) { + std::vector pieces; + pieces.reserve(completion); + for (size_t i = 0; i < completion; ++i) pieces.push_back(e.tok->decode(r.tokens[i])); + reasoning = think_reasoning_tokens(pieces, think_opened); + } + u["completion_tokens_details"] = {{"reasoning_tokens", reasoning}}; + } + return u; }; auto timings = [](const npu::GenerateResult& r, const std::string& cache) { json t{{"prompt_ms", r.prefill_ms}, diff --git a/cmake/tests-submodule-pin.cmake b/cmake/tests-submodule-pin.cmake new file mode 100644 index 000000000..3a00b8433 --- /dev/null +++ b/cmake/tests-submodule-pin.cmake @@ -0,0 +1,27 @@ +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# engine#2129: every submodule must be checked out at the commit the tree pins. +# +# A build compiles whatever is in the working tree, and every other check in tools/ci reads the tree it is handed - +# so none of them can see that a submodule sits on a different commit than the superproject pins. Measured on +# 2026-11-11: the shared ~/1bit-engine had four submodules off their pins (llama.cpp, hrx-system, tokenizers, +# xdna-driver), and the engine compiles common/jinja from llama.cpp - which was behind the pin on `\xNN` escapes. +# Jinja decides prompt rendering, so that is a difference in what the model sees, not a build failure. +# +# The default run compares only the submodules that ARE checked out (a configure that never inits them is not this +# check's business); `--require-initialised` is the release-gate form, where a missing checkout is a failure too. +add_python_test(submodule_pin_check ${CMAKE_SOURCE_DIR}/tools/ci/submodule_pin_check.py --check --root ${CMAKE_SOURCE_DIR}) +add_python_test(submodule_pin_check_self ${CMAKE_SOURCE_DIR}/tools/ci/submodule_pin_check.py --self-test) diff --git a/cmake/tests-tmp-path.cmake b/cmake/tests-tmp-path.cmake new file mode 100644 index 000000000..588193410 --- /dev/null +++ b/cmake/tests-tmp-path.cmake @@ -0,0 +1,24 @@ +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# engine#2021: a hardcoded `/tmp` path is RAM on this box, not disk. +# +# `/tmp` is tmpfs here, so a dump, a staging copy or a benchmark log written there is anonymous memory: swap fills +# and every lane on the box stalls (measured 2026-11-10 21:15Z: /tmp 52 G of 62 G, swap 71/71 GiB). engine#1907/#1952 +# bounded the fconst scratch; the literals themselves are still in the tree, 14 of them across 8 files. +# +# Both rows are cheap and device-free: the check is a grep over npu/ app/ tools/ scripts/ against +# tools/ci/tmp_paths.txt, and the ratchet may only shrink. +add_python_test(tmp_path_check ${CMAKE_SOURCE_DIR}/tools/ci/tmp_path_check.py --check --root ${CMAKE_SOURCE_DIR}) +add_python_test(tmp_path_check_self ${CMAKE_SOURCE_DIR}/tools/ci/tmp_path_check.py --self-test) diff --git a/config/llama-mirror-hold.config b/config/llama-mirror-hold.config new file mode 100644 index 000000000..b77bd4d8f --- /dev/null +++ b/config/llama-mirror-hold.config @@ -0,0 +1,34 @@ +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# The commit of 1bit-MONSTER/llama.cpp `1bit/hrx-vulkan` (our mirror of AMD's HRX line) that the daily +# bump-hrx run must not move away from. While a commit is named here, bump-hrx does not move the mirror, +# does not merge AMD's llama.cpp pin into `1bit/hrx-vulkan-patched`, reports the hold in the run summary and +# succeeds. Delete the commit line to lift the hold. +# +# The hold is for one situation only: AMD rewrote its line, so its pin no longer descends from the mirror, +# and taking it is a hand merge that has not been done yet. The run FAILS, naming this file, when the file +# is out of date (the mirror is no longer the named commit) or no longer needed (AMD's pin contains the +# named commit again), so a hold cannot outlive the state it describes. +# +# HELD 2026-11-11 for the release (orchestrator decision on engine#1945, the owner having delegated technical +# calls): the engine's llama.cpp pin stays where it is, and the merge of AMD's rebased line is post-release work. +# The state this records: +# mirror 1bit/hrx-vulkan 44ee5710d19f7f3af79b99bbcbe5fd13979d1f67 (AMD's line before its rebase) +# AMD pin 9bf5871310a8d7de74f4cb0c57009835fbf91c27 (the rebased line, 1777 upstream commits newer) +# merge into 1bit/hrx-vulkan-patched, against the last AMD commit our line took (43b0ec2c41cc): +# 149 conflicted files, 538 hunks (65 files / 83 hunks in the HRX backend) +# Lift it by deleting the line below, once AMD's line has been merged into ours. +44ee5710d19f7f3af79b99bbcbe5fd13979d1f67 diff --git a/docs/ci-runners.md b/docs/ci-runners.md index f3195c82a..107760817 100644 --- a/docs/ci-runners.md +++ b/docs/ci-runners.md @@ -306,7 +306,9 @@ here: a new workflow changes what the repository's token may do, which is an RFC ## Two rows, one question: are the runners healthy? Since #1682 and #1546 both landed there are two checks, and neither alone is enough — each is blind to what the -other sees. Both run in `scripts/weekly.sh`'s preflight, so a release refuses to start on a runner it cannot trust. +other sees. Both run in `scripts/weekly.sh`'s preflight. A runner that is **not accepting jobs** stops the release +before it starts. A runner that is up but whose **restart policy is short** is a warning, and the release goes on +(owner decision on engine#1504, 2026-11-11): see [the next section](#a-short-restart-policy-warns-it-does-not-block). | | `scripts/check-ci-runners.sh` | `scripts/ci-runner-health.sh` | |---|---|---| @@ -315,7 +317,7 @@ other sees. Both run in `scripts/weekly.sh`'s preflight, so a release refuses to | notices | a registration that was revoked, expired or never made | `Result=oom-kill`, a restart window, a unit that stopped | | cannot see | *why* it died — no `oom-kill`, no restart count | a unit that is running fine while its registration is gone | | speed | after the heartbeat times out | the moment the unit dies | -| exit | 0 online · 1 an offender, named · **2 cannot tell** (unreadable or truncated list) | 0 all accepting · 1 one is not · 2 no such units on this host | +| exit | 0 online · 1 an offender, named · **2 cannot tell** (unreadable or truncated list) | 0 all accepting, policy right · 1 one is not accepting jobs · 2 no such units on this host · **3 all accepting, a restart policy is short** | Run either by hand — both are read-only and neither restarts anything: @@ -335,3 +337,72 @@ rather than report green. One case neither row covers: a runner that is `online` in the API *and* active locally, but wedged so it takes no jobs. That is a hung process, not a registration or unit state; the practical signal for it is a queue that stops moving while both rows stay green. + +## A short restart policy warns, it does not block + +`scripts/ci-runner-health.sh` prints two lines per unit: its state (`PASS`/`FAIL`) and its restart policy +(`PASS`/`WARN`). The exit code keeps them apart, because they call for different things: + +| exit | meaning | `scripts/weekly.sh` preflight | +|---|---|---| +| 0 | every unit is accepting jobs and would come back after an OOM kill | passes | +| 1 | a unit is not accepting jobs now (dead, or in its restart window) | **blocks**: the checks after the release would queue for ever | +| 2 | no `actions.runner.*` units on this host | a note | +| 3 | every unit is accepting jobs, but a `WARN` line names a policy setting | **warns** and goes on | + +A unit that is down outranks a policy line, so 1 wins over 3. The warning in the release log reads +`preflight: WARNING, not blocking: every runner unit is accepting jobs, but a restart policy would leave one down +after the next OOM kill`, with the `WARN` lines above it naming the unit and the setting. + +Why a warning is enough: the policy is a box configuration change, which the owner applies (below), and a release +that cannot start does not make the unit any safer. Meanwhile the heavy builds run under +`scripts/mem-guard.sh --term`, which ends the build before the kernel's OOM killer reaches the runner. **A build step +that ends with exit 143 was stopped by that guard**: the log has a `GUARD: MemAvailable N GB < floor GB` line, the +runner is still alive, and the job is re-run once memory frees. The guard covers only the steps that use it. A server +or a bench started without it can still take the unit down, which is why the drop-ins below are the real fix. + +`tests/weekly_runner_preflight_test.sh` (ctest `weekly_runner_preflight`) runs the real health script and the real +`preflight_runner_units()` from `scripts/weekly.sh` against a stub `systemctl`: the four exit codes, block on a dead +unit, warn on a short policy. + +### The two drop-ins (owner step) + +On strixhalo, `strixhalo-bench` and `strixhalo-org` still run the units that predate `scripts/install-ci-runner.sh`: +`Restart=no`, `OOMPolicy=stop`. One OOM kill of any process in the unit stops it for good. `strixhalo-engine` got +the same drop-in on 2026-11-10. This needs `sudo`, so it is the owner's to run. It does not re-register a runner, +change its user, or stop a job in flight. + +Both files have the same content, [`scripts/ci-runner-restart.conf`](../scripts/ci-runner-restart.conf): + +```ini +# /etc/systemd/system/actions.runner.1bit-MONSTER-engine.strixhalo-bench.service.d/restart.conf +# /etc/systemd/system/actions.runner.1bit-MONSTER.strixhalo-org.service.d/restart.conf +[Unit] +StartLimitIntervalSec=0 + +[Service] +Restart=always +RestartSec=15 +OOMPolicy=continue +``` + +The three commands, from the root of an engine checkout: + +```sh +sudo install -Dm644 scripts/ci-runner-restart.conf /etc/systemd/system/actions.runner.1bit-MONSTER-engine.strixhalo-bench.service.d/restart.conf +sudo install -Dm644 scripts/ci-runner-restart.conf /etc/systemd/system/actions.runner.1bit-MONSTER.strixhalo-org.service.d/restart.conf +sudo systemctl daemon-reload +``` + +Check it took (read-only, no `sudo`): + +```sh +systemctl show actions.runner.1bit-MONSTER-engine.strixhalo-bench.service actions.runner.1bit-MONSTER.strixhalo-org.service -p Id -p Restart -p OOMPolicy +scripts/ci-runner-health.sh; echo "exit $?" +``` + +Want `Restart=always` and `OOMPolicy=continue` on both. The health script still exits 3 afterwards, with one `WARN` +per unit naming only `MemoryMax=infinity(want a ceiling)`: the drop-in sets no memory ceiling. The ceiling this page +documents (`MemoryHigh=43G`, `MemoryMax=48G`) comes with a full `scripts/install-ci-runner.sh` install, or with +`sudo systemctl set-property MemoryHigh=43G MemoryMax=48G` per unit. To undo a drop-in, delete the +`restart.conf` and run `sudo systemctl daemon-reload`. diff --git a/docs/hrx.md b/docs/hrx.md index 0b1ebecee..d9feccefd 100644 --- a/docs/hrx.md +++ b/docs/hrx.md @@ -98,6 +98,19 @@ fails a pin that leaves a gate model's node off HRX. `ggml/src/ggml-hrx` keep our side; a conflict anywhere else fails the run and names the files; - merges AMD's hrx-system pin into our `1bit/main` the same way; - opens a PR here moving the submodules, listing the llama.cpp commits new to the engine. +- **The mirror hold.** A rewrite of AMD's line that the rules above cannot merge leaves the daily run with + nothing to do but fail, every day, until somebody does the merge by hand. While + [`config/llama-mirror-hold.config`](../config/llama-mirror-hold.config) names the mirror's commit, the run + instead reports the hold in its summary (`mirror 1bit/hrx-vulkan HELD at …; AMD's pin is …, on a line that + does not contain it`), moves nothing and succeeds. The engine's pin does not move in a held run either: moving + it along our own line is a reviewed PR of its own. The hold + describes one state and fails the run when that state ends, so it cannot be forgotten: **stale** when the + mirror is no longer the named commit, **unneeded** when AMD's pin contains the named commit again (an ordinary + sync would work). `scripts/bump-hrx-sync.sh llama-hold ` prints the decision alone + (`none`, `held`, `stale`, `unneeded`). **Held since 2026-11-11** for the release (engine#1945): the file names + the mirror's commit `44ee5710d19f`, and the merge of AMD's rebased line is post-release work. It was added for AMD's 2026-11-09 rebase (engine#1945): taking that pin is a merge of + 149 conflicted files (538 hunks) into `1bit/hrx-vulkan-patched`, measured against the last AMD commit our + line took. - **The hold.** While [`config/hrx-system-hold.config`](../config/hrx-system-hold.config) names a commit, the run does not merge AMD's hrx-system, keeps `third_party/hrx-system` where it is, and takes only AMD's llama.cpp changes outside the HRX backend paths @@ -377,6 +390,38 @@ layout (8.80 s and 8.85 s) and gives the same text. `test-hrx-fa-transposed-valu backend (8 cases). Why 256-token launches are faster than one 512-token launch at long key counts is measured, not explained. +### Fixed: a gated-delta-net layer behind a type cast failed the whole graph ([engine#1925](https://github.com/1bit-MONSTER/engine/issues/1925)) + +On Qwen3.8-Flash-Next, a long prompt failed at the first 512-token batch past 2,048 tokens of context when the +sparse attention path's mask chain ran on the device: + +``` +graph_compute: unsupported HRX node 160: L2_NORM output=f32[128,16,512,1]<-L2_NORM inputs=[f32[128,16,512,1]<-VIEW] consumers=[174:GATED_DELTA_NET] +``` + +The fused gated-delta-net dispatch did match that node. The fault was the order in which the graph was walked. +The scheduler walks a graph in dependency order and takes the matmuls first, which is what puts a layer's gate +and beta projections ahead of the `L2_NORM` its fused pattern starts from. A type cast is an in-place copy that +names its own output as an input. The walk counted that as a dependency, so the copy waited for itself, the walk +stopped there, and every node after it was taken in index order. In index order the `L2_NORM` comes before the +projections, and the pattern was refused because it would have read a projection that had not run. The sparse +path's mask chain contains such casts, so the failure needed both a long context and that chain on the device. + +The walk no longer counts a node's own output as a dependency (llama.cpp #302). The fused matcher also keeps a +match when a node of its pattern that only renames storage was already passed. + +Measured on gfx1151 (2026-11-11, Flash-Next UD-IQ3_XXS, `-c 19456 -np 1`, one 18,782-token request, a build of +the fork with only the two changed objects replaced): `unsupported HRX node` 1 before, 0 after; all 390 fused +gated-delta-net roots accepted; the whole prefill runs. `tests/test-hrx-gdn-passed-alias.cpp` plans the layer +behind a cast on the host and requires the same dispatches as without it. Qwen3.5-4B, which has the same layer +type and no sparse path, gives the same tokens on the old and the new library (128 of 128 in three alternated pairs), uses the same 33 +dispatch registrations in the same order, and runs at the same rate (`llama-bench -p 512 -n 64 -r 5`, old, new, +old, new: decode 57.4, 58.3, 58.1, 58.2 tok/s; prefill 3,516, 3,042, 3,109, 3,213). + +Not shown by this: that the answer at 18.8k is the same with the mask chain on the device as with it on the +CPU. Three of the four long requests in these runs, on the old library and the new, ended in all-NaN logits on +the first sampled token, which is [engine#1536](https://github.com/1bit-MONSTER/engine/issues/1536). + ### MoE decode and the kernel cache (llama.cpp #32, #33) - **MoE decode.** With one to four tokens, a `MUL_MAT_ID` (a MoE expert matmul) runs as a GEMV over diff --git a/docs/hw-gate.md b/docs/hw-gate.md index 1d7afdeef..ed0dedb4a 100644 --- a/docs/hw-gate.md +++ b/docs/hw-gate.md @@ -72,7 +72,9 @@ hrx-system) through `cmake/hrx.cmake`, then links `onebit` with its HRX wiring. - **Memory.** The box is shared, and on Strix Halo GTT is host RAM. Before it compiles, the job waits for 24 GB of `MemAvailable` (`scripts/mem-wait.sh`, polled once a minute for up to 15 minutes). If memory doesn't free up, the job fails with the reading, and a re-run is the fix. The compile itself runs under `scripts/mem-guard.sh --term 12`, - which stops the whole `thermal-run` scope if `MemAvailable` falls below 12 GB. On 2026-11-08 both self-hosted + which stops the whole `thermal-run` scope if `MemAvailable` falls below 12 GB. **Exit 143 in a build step is + that guard, not a compiler failure**: the step log has a `GUARD: MemAvailable N GB < 12 GB` line just above it, the + runner is still alive, and a re-run once memory frees is the fix. On 2026-11-08 both self-hosted runners were OOM-killed, the second time 76 s after a restart handed this job a compile (engine#1504). - **What it is not.** It is a compile and link check of the public plumbing only. It does not build the private HIP add-on (the gate builds that) and does not dispatch anything. Green means the pinned pair builds into the engine. It @@ -250,6 +252,87 @@ The settings are environment variables at the top of the script: `HW_GATE_STORE` `$HW_GATE_OUT` (default `~/.cache/hw-gate/runs/