From e4cea323eb74d3e960620c963d23ef9d2b62330a Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 8 Nov 2026 09:55:11 -0400 Subject: [PATCH] npu(p4): the lm-head context is the target's, so the forward no longer creates one (A2b, last setup block) The block the previous commit cut out of init_impl() moves behind the target, body verbatim: the pb16i ternary-image branch, the int8 ladder, the q4_1 gemv head, every init_elf call and every refusal message. With this, a grep for make_head_ctx in q4nx_forward.cpp is empty -- the forward creates no I8Ctx at all any more; the prefill set moved in #1380 and the head moves here. What stayed in the forward, deliberately, and arrives through Setup: the container's NAMING. Deciding whether the head's ternary image exists at all is a path rule built from the head's sidecar stem and the image extension, which is this file's knowledge and not a target's, so the forward computes a head_pb16i_images boolean and passes the sidecar directory, the decode-dir list and the head geometry (hidden, vocab, kpad). A target that recomputed that rule would be a second copy of the container's naming, which is how a plan and its images drift apart. head_pb16i_ stays a forward flag -- the model path reads it in four places -- and the target writes its own copy which the forward reads back through head_is_pb16i(), so which leg opened the head is still visible where it matters. Gate: ctest 128/128 on a full Release build (ONEBIT_NPU=ON, runtime from ONEBIT_NPU_RT_DIR, under thermal-run.sh). The device half is the folded 27B pb16i row, A = main and B = this head, requiring the same text and the same 582/582 pb16i trace and 14/14 qkv taps: this block runs ONLY for a pb16i container and the 0.6B on this box is rm, so no ctest reaches it, and a build-only claim would be claiming the one path it cannot reach. That row is armed with a watcher because the box is at load 30+ with another lane holding the device. --- npu/forward/include/q4nx_forward.h | 3 - npu/forward/src/q4nx_forward.cpp | 157 +++-------------------------- npu/target/xdna/xdna_target.cpp | 137 +++++++++++++++++++++++++ npu/target/xdna/xdna_target.h | 19 ++++ 4 files changed, 170 insertions(+), 146 deletions(-) diff --git a/npu/forward/include/q4nx_forward.h b/npu/forward/include/q4nx_forward.h index 8535c21d7..e1fad2d3d 100644 --- a/npu/forward/include/q4nx_forward.h +++ b/npu/forward/include/q4nx_forward.h @@ -334,9 +334,6 @@ class Q4nxNpuForward { bool noncausal_attn_ = false; // The refusal for a kernel a design needs but the directory does not have. With a plan it names // the plan file and the role, so "which description asked for this" needs no log reading. - // The lm-head context: pb16i ternary images, the int8 ladder, or the q4_1 gemv head. Extracted from - // init_impl() with its body unchanged (engine#803 A2b), so moving it behind XdnaTarget is a copy. - bool open_lm_head_context(xrt::device* dev, const std::string& model_dir, const std::string& kernels_dir); std::string missing_kernel(const std::string& dir, const char* elf_name, Q4nxDesign role) const; // Index of the design playing a role, or -1 when the model has no such // kernel (e.g. GU is absent on split models, G/U absent on fused ones). diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index fd3217911..724f66865 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -1114,6 +1114,15 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, setup.t2s = t2s_; setup.t2s_v2 = t2s_v2_; setup.host_gemm = host_gemm_; + // The lm-head block's container-side inputs: the sidecar dir, whether the head's ternary image is + // there at all (this file owns that path rule), where decode twins are looked for, and the geometry. + setup.pb16i_dir = pb16i_dir_; + setup.head_pb16i_images = + pb16i_ && !pb16i_dir_.empty() && file_exists(head_pb16i_stem(0) + image_ext()); + setup.decode_dirs = m16dirs_like_head(mdir, dir, plan_decode_dirs_); + setup.hidden = c_.H; + setup.vocab = c_.NV; + setup.kpad = kpad_; setup.plan = plan_.loaded() ? &plan_ : nullptr; xd_->configure(setup); std::string why; @@ -1320,7 +1329,11 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, // The lm head's own context (pb16i ternary images, the int8 ladder, or the q4_1 gemv head). Cut out // into its own method with its body unchanged, so the next step (moving it behind XdnaTarget, as the // prefill set already is) is a copy rather than a surgery inside init_impl. - if (!open_lm_head_context(dev, mdir, dir)) return false; + // The lm-head context is the target's now (engine#803 A2b): the body moved verbatim into + // XdnaTarget::open_head(), and what stayed here is the container's naming, because which sidecar file + // the head has is the container's rule. The flag the model path reads is written by the target. + if (!xd_->open_head(designs_, ndesigns_, err_)) return false; + head_pb16i_ = xd_->head_is_pb16i(); // ---- Gemma4 per-layer embeddings (PLE) + KV sharing ---- // Every tensor here is read out of the model's own index (name + dtype checked @@ -3532,148 +3545,6 @@ bool Q4nxNpuForward::step_batch(const std::vector& tokens, int start_pos, } -// The lm-head context, moved out of init_impl() unchanged (engine#803 A2b, step 1 of 2): pb16i ternary -// images first, then the int8 ladder, then the q4_1 gemv head. No logic changed; the body is the block -// that used to sit inline, so the device behaviour is the same code. -bool Q4nxNpuForward::open_lm_head_context(xrt::device* dev, const std::string& mdir, - const std::string& dir) { - // run_lm_head packs int8 B tiles from the f32 head and launches them itself, so it - // needs an int8 context with K=H. Under pb16i every decode context is a pb16i one, - // so build it here: the widest K=H design, at M=16 if its int8 twin is on disk. - if (pb16i_ && !host_gemm_) { - int i_head = -1; - for (int i = 0; i < ndesigns_; i++) { - if (!xd_->ctx(i) || designs_[i].K != c_.H || designs_[i].N <= 0) continue; - if (i_head < 0 || designs_[i].N > designs_[i_head].N) i_head = i; - } - // The head's ternary images come first, when the converter produced them: output.weight - // is PTQ1_0 (type 143) in the folded GGUF, so the image is exact rather than a - // requantisation, it is 0.75 GB against the int8 head's 1.33 GB, and it runs the same - // K=H kernel the MLP's GU launch runs. Missing images are not an error: the head then - // keeps the int8 family below. - if (i_head >= 0 && !pb16i_dir_.empty() && - file_exists(head_pb16i_stem(0) + image_ext())) { - const Q4nxDesignGeom& g = designs_[i_head]; - std::string stem(g.elf_name); - stem.resize(stem.size() - 4); // drop ".elf" - const std::string ft = family_tag(); - const std::string cands[] = {stem + ft + "_m16.elf", stem + ft + ".elf"}; - std::string pelf; - for (const std::string& c : cands) { - for (const std::string& d : m16dirs_like_head(mdir, dir)) { - if (file_exists(d + "/" + c)) { pelf = d + "/" + c; break; } - } - if (!pelf.empty()) break; - } - if (!pelf.empty()) { - I8Ctx* hc = xd_->make_head_ctx(); - if (!hc) { err_ = "cannot create the lm-head context"; return false; } - hc->MD = Q4NX_DESIGN_M_DECODE; - hc->KD = g.K; - hc->ND = g.N; - hc->pb16i = true; - hc->t2s = t2s_; - hc->pb16i_planes = 1; // the K=H design is the arena's one-plane family - if (hc->init_elf(*dev, pelf.c_str(), Q4NX_DESIGN_M_DECODE, g.K, g.N, 1)) { - head_pb16i_ = true; - fprintf(stderr, " [forward] lm head M=%d K=%d N=%d %s (pb16i ternary " - "images)\n", hc->MD, g.K, g.N, pelf.c_str()); - } else { - xd_->head_ctx_ref().reset(); - } - } - } - if (!xd_->head_ctx() && i_head >= 0) { - const Q4nxDesignGeom& g = designs_[i_head]; - // The head streams 1.33 GB of int8 weights a token, so its layout matters more - // than its row count: a blocked (_b8) ELF walks 8 KiB forward per tile at - // 8.25 GB/s where the row-major one runs 128 separate 128-byte reads at - // 3.3 GB/s (measured: 27 ms a launch either way for M=16 and M=128). Try the - // blocked M=16 twin first, then whatever int8 family is on disk. - std::string stem(g.elf_name); - stem.resize(stem.size() - 4); // drop ".elf" - const std::string cands[] = {stem + "_b8_m16.elf", stem + "_m16.elf", - stem + "_b8.elf", g.elf_name}; - std::string elf; - bool blocked = weight_layout_b8_; - for (const std::string& c : cands) { - for (const std::string& d : m16dirs_like_head(mdir, dir)) { - if (file_exists(d + "/" + c)) { elf = d + "/" + c; break; } - } - if (!elf.empty()) { - blocked = c.find("_b8") != std::string::npos; - break; - } - } - int m = elf.find("_m16.elf") != std::string::npos ? Q4NX_DESIGN_M_DECODE : g.M; - if (elf.empty()) { elf = dir + "/" + g.elf_name; m = g.M; } - I8Ctx* hc = xd_->make_head_ctx(); - if (!hc) { err_ = "cannot create the lm-head context"; return false; } - hc->MD = m; hc->KD = g.K; hc->ND = g.N; - hc->blocked_b = blocked; - if (!hc->init_elf(*dev, elf.c_str(), m, g.K, g.N, 1)) { - fprintf(stderr, " [forward] lm head: cannot load %s; the head keeps the " - "decode family\n", elf.c_str()); - xd_->head_ctx_ref().reset(); - } else { - fprintf(stderr, " [forward] lm head M=%d K=%d N=%d %s\n", m, g.K, g.N, - elf.c_str()); - } - } - } - - // ---- the lm head on the q4_1 gemv family ---- - // That family has no int8 packer: its weight BO holds the container's own q4_1 tile stream and the - // launch dequantises from those tiles, so the tiled int8 head above cannot express it -- packing an - // int8 image into a tile BO both overruns the BO by (KD*ND - tile bytes) and means nothing to the - // kernel. The head is therefore ONE launch whose N is the vocabulary: the only width whose output - // is the whole logits row, which is what run_lm_head_gemv() gathers the container's head tiles for. - if (!host_gemm_ && !xd_->head_ctx()) { - bool any_gemv = false; - for (int i = 0; i < ndesigns_ && !any_gemv; i++) - any_gemv = (xd_->ctx(i) && xd_->ctx(i)->gemv) || (xd_->ctx16(i) && xd_->ctx16(i)->gemv); - if (any_gemv) { - // The head ELF's N is on the gemv column grid, so try the padded vocabulary first and the - // raw one after: the pack's vocab 151936 is 152064 on the grid. - std::string elf; - for (int nn : {q4nx_pad_n(c_.NV), c_.NV}) { - for (const char* suf : {"_m16.elf", ".elf"}) { - const std::string n = "full_i8_LMHEAD_K" + std::to_string(kpad_) + "_N" + - std::to_string(nn) + suf; - for (const std::string& d : m16dirs_like_head(mdir, dir, plan_decode_dirs_)) { - if (file_exists(d + "/" + n)) { elf = d + "/" + n; break; } - } - if (!elf.empty()) break; - } - if (!elf.empty()) break; - } - if (elf.empty()) { - err_ = "lm head: this container's kernels are the q4_1 gemv family, whose head is one " - "launch at N=vocab, and no head ELF is on disk (want full_i8_LMHEAD_K" + - std::to_string(kpad_) + "_N" + std::to_string(q4nx_pad_n(c_.NV)) + "[_m16].elf)"; - return false; - } - const int m = elf.find("_m16.elf") != std::string::npos ? Q4NX_DESIGN_M_DECODE - : Q4NX_DESIGN_M_PREFILL; - const int n = (elf.find("_N" + std::to_string(q4nx_pad_n(c_.NV))) != std::string::npos) - ? q4nx_pad_n(c_.NV) : c_.NV; - I8Ctx* hc = xd_->make_head_ctx(); - if (!hc) { err_ = "lm head: cannot create the gemv head context"; return false; } - hc->MD = m; hc->KD = kpad_; hc->ND = n; - if (!hc->init_elf(*dev, elf.c_str(), m, kpad_, n, 1) || !hc->gemv) { - err_ = "lm head: " + elf + " does not load as the gemv head (K=" + - std::to_string(kpad_) + " N=" + std::to_string(n) + ")"; - xd_->head_ctx_ref().reset(); - return false; - } - fprintf(stderr, " [forward] lm head M=%d K=%d N=%d %s (q4_1 gemv tiles)\n", m, kpad_, n, - elf.c_str()); - } - } - - return true; -} - std::string Q4nxNpuForward::missing_kernel(const std::string& dir, const char* elf_name, Q4nxDesign role) const { if (plan_.loaded()) diff --git a/npu/target/xdna/xdna_target.cpp b/npu/target/xdna/xdna_target.cpp index 3f09b5cba..dc8f2b4cc 100644 --- a/npu/target/xdna/xdna_target.cpp +++ b/npu/target/xdna/xdna_target.cpp @@ -485,6 +485,143 @@ void XdnaTarget::prof_dump() { prof_head_ = ProfAcc{}; } +bool XdnaTarget::open_head(const Q4nxDesignGeom* designs, int n, std::string& err) { + // run_lm_head packs int8 B tiles from the f32 head and launches them itself, so it + // needs an int8 context with K=H. Under pb16i every decode context is a pb16i one, + // so build it here: the widest K=H design, at M=16 if its int8 twin is on disk. + if (setup_.pb16i && !setup_.host_gemm) { + int i_head = -1; + for (int i = 0; i < n; i++) { + if (!ctx(i) || designs[i].K != setup_.hidden || designs_[i].N <= 0) continue; + if (i_head < 0 || designs[i].N > designs[i_head].N) i_head = i; + } + // The head's ternary images come first, when the converter produced them: output.weight + // is PTQ1_0 (type 143) in the folded GGUF, so the image is exact rather than a + // requantisation, it is 0.75 GB against the int8 head's 1.33 GB, and it runs the same + // K=H kernel the MLP's GU launch runs. Missing images are not an error: the head then + // keeps the int8 family below. + if (i_head >= 0 && !setup_.pb16i_dir.empty() && setup_.head_pb16i_images) { + const Q4nxDesignGeom& g = designs[i_head]; + std::string stem(g.elf_name); + stem.resize(stem.size() - 4); // drop ".elf" + const std::string ft = setup_.family; + const std::string cands[] = {stem + ft + "_m16.elf", stem + ft + ".elf"}; + std::string pelf; + for (const std::string& c : cands) { + for (const std::string& d : setup_.decode_dirs) { + if (elf_present(d + "/" + c)) { pelf = d + "/" + c; break; } + } + if (!pelf.empty()) break; + } + if (!pelf.empty()) { + I8Ctx* hc = make_head_ctx(); + if (!hc) { err = "cannot create the lm-head context"; return false; } + hc->MD = Q4NX_DESIGN_M_DECODE; + hc->KD = g.K; + hc->ND = g.N; + hc->pb16i = true; + hc->t2s = setup_.t2s; + hc->pb16i_planes = 1; // the K=H design is the arena's one-plane family + if (hc->init_elf(*dev_, pelf.c_str(), Q4NX_DESIGN_M_DECODE, g.K, g.N, 1)) { + head_pb16i_ = true; + fprintf(stderr, " [forward] lm head M=%d K=%d N=%d %s (pb16i ternary " + "images)\n", hc->MD, g.K, g.N, pelf.c_str()); + } else { + head_ctx_ref().reset(); + } + } + } + if (!head_ctx() && i_head >= 0) { + const Q4nxDesignGeom& g = designs[i_head]; + // The head streams 1.33 GB of int8 weights a token, so its layout matters more + // than its row count: a blocked (_b8) ELF walks 8 KiB forward per tile at + // 8.25 GB/s where the row-major one runs 128 separate 128-byte reads at + // 3.3 GB/s (measured: 27 ms a launch either way for M=16 and M=128). Try the + // blocked M=16 twin first, then whatever int8 family is on disk. + std::string stem(g.elf_name); + stem.resize(stem.size() - 4); // drop ".elf" + const std::string cands[] = {stem + "_b8_m16.elf", stem + "_m16.elf", + stem + "_b8.elf", g.elf_name}; + std::string elf; + bool blocked = setup_.b8; + for (const std::string& c : cands) { + for (const std::string& d : setup_.decode_dirs) { + if (elf_present(d + "/" + c)) { elf = d + "/" + c; break; } + } + if (!elf.empty()) { + blocked = c.find("_b8") != std::string::npos; + break; + } + } + int m = elf.find("_m16.elf") != std::string::npos ? Q4NX_DESIGN_M_DECODE : g.M; + if (elf.empty()) { elf = setup_.kernels_dir + "/" + g.elf_name; m = g.M; } + I8Ctx* hc = make_head_ctx(); + if (!hc) { err = "cannot create the lm-head context"; return false; } + hc->MD = m; hc->KD = g.K; hc->ND = g.N; + hc->blocked_b = blocked; + if (!hc->init_elf(*dev_, elf.c_str(), m, g.K, g.N, 1)) { + fprintf(stderr, " [forward] lm head: cannot load %s; the head keeps the " + "decode family\n", elf.c_str()); + head_ctx_ref().reset(); + } else { + fprintf(stderr, " [forward] lm head M=%d K=%d N=%d %s\n", m, g.K, g.N, + elf.c_str()); + } + } + } + + // ---- the lm head on the q4_1 gemv family ---- + // That family has no int8 packer: its weight BO holds the container's own q4_1 tile stream and the + // launch dequantises from those tiles, so the tiled int8 head above cannot express it -- packing an + // int8 image into a tile BO both overruns the BO by (KD*ND - tile bytes) and means nothing to the + // kernel. The head is therefore ONE launch whose N is the vocabulary: the only width whose output + // is the whole logits row, which is what run_lm_head_gemv() gathers the container's head tiles for. + if (!setup_.host_gemm && !head_ctx()) { + bool any_gemv = false; + for (int i = 0; i < ndesigns_ && !any_gemv; i++) + any_gemv = (ctx(i) && ctx(i)->gemv) || (ctx16(i) && ctx16(i)->gemv); + if (any_gemv) { + // The head ELF's N is on the gemv column grid, so try the padded vocabulary first and the + // raw one after: the pack's vocab 151936 is 152064 on the grid. + std::string elf; + for (int nn : {q4nx_pad_n(setup_.vocab), setup_.vocab}) { + for (const char* suf : {"_m16.elf", ".elf"}) { + const std::string n = "full_i8_LMHEAD_K" + std::to_string(setup_.kpad) + "_N" + + std::to_string(nn) + suf; + for (const std::string& d : setup_.decode_dirs) { + if (elf_present(d + "/" + n)) { elf = d + "/" + n; break; } + } + if (!elf.empty()) break; + } + if (!elf.empty()) break; + } + if (elf.empty()) { + err = "lm head: this container's kernels are the q4_1 gemv family, whose head is one " + "launch at N=vocab, and no head ELF is on disk (want full_i8_LMHEAD_K" + + std::to_string(setup_.kpad) + "_N" + std::to_string(q4nx_pad_n(setup_.vocab)) + "[_m16].elf)"; + return false; + } + const int m = elf.find("_m16.elf") != std::string::npos ? Q4NX_DESIGN_M_DECODE + : Q4NX_DESIGN_M_PREFILL; + const int n = (elf.find("_N" + std::to_string(q4nx_pad_n(setup_.vocab))) != std::string::npos) + ? q4nx_pad_n(setup_.vocab) : setup_.vocab; + I8Ctx* hc = make_head_ctx(); + if (!hc) { err = "lm head: cannot create the gemv head context"; return false; } + hc->MD = m; hc->KD = setup_.kpad; hc->ND = n; + if (!hc->init_elf(*dev_, elf.c_str(), m, setup_.kpad, n, 1) || !hc->gemv) { + err = "lm head: " + elf + " does not load as the gemv head (K=" + + std::to_string(setup_.kpad) + " N=" + std::to_string(n) + ")"; + head_ctx_ref().reset(); + return false; + } + fprintf(stderr, " [forward] lm head M=%d K=%d N=%d %s (q4_1 gemv tiles)\n", m, setup_.kpad, n, + elf.c_str()); + } + } + + return true; +} + std::string XdnaTarget::pb16i_stem(int layer, const char* tag) const { return pb16i_dir_ + "/" + q4nx_pb16i::sidecar_stem(layer, tag); } diff --git a/npu/target/xdna/xdna_target.h b/npu/target/xdna/xdna_target.h index baf65df84..3ba8ae2ce 100644 --- a/npu/target/xdna/xdna_target.h +++ b/npu/target/xdna/xdna_target.h @@ -128,6 +128,14 @@ class XdnaTarget final : public NpuTarget { bool t2s = false; // 2-bit ternary stream bool t2s_v2 = false; bool host_gemm = false; // offline: open no ELF at all + // The lm-head context needs the container's own naming, which is the forward's to compute and not + // the target's: the sidecar directory, whether the head's ternary image exists at all (the test is + // `head_pb16i_stem(0) + image_ext()` on the forward's side), and where decode twins are looked for. + std::string pb16i_dir; + bool head_pb16i_images = false; + std::vector decode_dirs; + // The head geometry: hidden width, vocabulary, and the container's padded K. + int hidden = 0, vocab = 0, kpad = 0; const onebit::npu::plan::Plan* plan = nullptr; // copied into plan_ by configure(); null = none }; // Setup carries a pointer to the forward's Plan; this copies it, so from here on the target's own @@ -149,6 +157,16 @@ class XdnaTarget final : public NpuTarget { // means no plan, i.e. a purely derived set). bool open_prefill(const Q4nxDesignGeom* designs, int n, std::string& err); + // The lm-head context (engine#803 A2b, the last setup block): pb16i ternary images first, then the int8 + // ladder, then the q4_1 gemv head -- the body moved verbatim out of Q4nxNpuForward::open_lm_head_context. + // It is a context and its refusals, which is what a target owns; the container's *naming* stayed in the + // forward and arrives through Setup, because deciding which sidecar file the head has is the container's + // rule and not a target's. + bool open_head(const Q4nxDesignGeom* designs, int n, std::string& err); + // Whether open_head() found and opened the pb16i ternary head. The model path reads this flag in four + // places, so it stays the forward's; the target writes it. + bool head_is_pb16i() const { return head_pb16i_; } + // Context construction: the caller configures and initialises the I8Ctx, then it stays here. I8Ctx* make_ctx(int role); I8Ctx* ctx(int role) const; @@ -189,6 +207,7 @@ class XdnaTarget final : public NpuTarget { std::string pb16i_stem(int layer, const char* tag) const; xrt::device* dev_ = nullptr; + bool head_pb16i_ = false; // written by open_head(), read back by the forward Setup setup_; // see configure(): what the setup side reads, in one place int n_layers_ = 0; std::string kernels_dir_;