From a8c378f77bcd24ccc88388a75f5977a260d3b1fb Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Mon, 9 Nov 2026 21:06:10 -0400 Subject: [PATCH 1/2] npu: the M=16 decode family is the target's -- open_decode_family, moved verbatim (engine#803 A2b) The last structural block of A2: `init_impl` no longer decides which decode contexts exist and configures them field by field. `XdnaTarget::open_decode_family()` does, from the same inputs and with the same messages: - resolves each role's `_m16` ELF under the plan's decode directories (setup_.decode_dirs), or the sealed pb16i image with its instance name from q4nx_pb16i_instance (hoisted in #1765 for exactly this); - a pb16i role with no image STOPS the load by name (the int8 weights are not packed for those geometries under this layout); every other role without an ELF keeps M=128, which is a slower path and not a missing one; - creates the context and sets MD/KD/ND, blocked_b, pb16i, t2s and pb16i_planes (the ACC64 two-plane family: D, O and the full-attention q half), and on an init failure drops the context and keeps M=128 unless pb16i; - zeroes `designs16[i].M` for a role it cannot serve -- the design itself carries "no decode design", which is what the launch path reads. What stayed in the forward is the container's naming (`pb16i_sealed_`: which sidecar file the sealed ELF is) and the two summary lines, because they are about the model's geometry rather than a target's artefacts. The log lines moved verbatim (`[forward] ROLE M=.. K=.. N=.. (decode)`, the "M=16 twin failed to load" line, the pb16i "no instance for role" line) because the window11 and [pb16i-trace] gates grep them. Verified: an ONEBIT_NPU=ON `1bit` builds and links; npu_pack_test ok; npu_forward_loader_test ok. NOT verified: the device row -- window11 text identity against a pristine-main arm, which is the gate this move has to pass before merge, on bonsai2-27b-NPU (both kernel families are on the box). (cherry picked from commit 7e60e729e18886e4b30d53601400c19394b9755c) --- npu/forward/src/q4nx_forward.cpp | 97 ++------------------------------ npu/target/xdna/xdna_target.cpp | 96 +++++++++++++++++++++++++++++++ npu/target/xdna/xdna_target.h | 11 ++++ 3 files changed, 112 insertions(+), 92 deletions(-) diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 09cba1ef7..eb066050b 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -1311,99 +1311,12 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, snprintf(designs16_[i].elf_name, Q4NX_ELF_NAME_MAX, "%s", n16.c_str()); } } - std::vector m16dirs; - plan_decode_dirs_.clear(); - if (plan_.loaded() && !plan_.decode_dirs.empty()) { - // The plan says which of the two given paths each candidate hangs off, so the runtime - // stops guessing which base a bare directory name meant. - for (const auto& d : plan_.decode_dirs) - plan_decode_dirs_.push_back((d.under_model ? mdir : dir) + "/" + d.path); - for (const auto& p : plan_decode_dirs_) m16dirs.push_back(p); - } else { - m16dirs.push_back(mdir + "/kernels-m16"); - m16dirs.push_back(dir + "/m16"); - m16dirs.push_back(dir); - } + // The decode family is the target's (engine#803 A2b): it resolves each role's `_m16` ELF under the plan's + // decode directories (or the sealed pb16i image), creates the context and configures it. What stays here is + // the container's naming -- which sidecar file the sealed ELF is -- and the summary line, which is about the + // model's geometry rather than the target's. int made = 0, skipped = 0; - for (int i = 0; i < ndesigns_; i++) { - if (designs16_[i].M <= 0) { skipped++; continue; } - std::string elf, kname = "main:seq"; - if (pb16i_ && !pb16i_sealed_.empty()) { - // One ELF for every ternary phase: the file is the same for all roles and - // only the instance name differs, so I8Ctx reuses the one hw_context. - const char* inst = pb16i_instance(designs16_[i].role); - if (!inst) { - fprintf(stderr, " [forward] pb16i: the sealed ELF has no instance for " - "role %s; this role has no decode kernel\n", - role_tag(designs16_[i].role)); - designs16_[i].M = 0; - skipped++; - continue; - } - elf = dir + "/" + pb16i_sealed_; - kname = inst; - if (!file_exists(elf)) { - err_ = plan_.loaded() ? ("plan " + plan_path_ + ": decode.sealed_elf \"" + - pb16i_sealed_ + "\" is not in " + dir + " (missing)") - : ("pb16i: missing sealed ELF " + elf); - return false; - } - } else { - for (const std::string& d : m16dirs) { - std::string cand = d + "/" + designs16_[i].elf_name; - if (file_exists(cand)) { elf = cand; break; } - } - } - if (elf.empty()) { - // A pb16i role without its image kernel must stop the load: it is not - // a slower path, it is a missing one (the int8 weights are not packed - // for these geometries under this layout). - if (pb16i_) { - err_ = plan_.loaded() - ? std::string("plan ") + plan_path_ + ": decode role " + - role_tag(designs16_[i].role) + " needs \"" + designs16_[i].elf_name + - "\" (missing)" - : std::string("pb16i: missing decode ELF ") + designs16_[i].elf_name; - return false; - } - skipped++; - continue; // stay on M=128 - } - I8Ctx* c16 = xd_->make_ctx16(i, designs16_[i]); - if (!c16) { err_ = "cannot create the decode context for role " + std::to_string(i); return false; } - c16->MD = designs16_[i].M; - c16->KD = designs16_[i].K; - c16->ND = designs16_[i].N; - c16->blocked_b = weight_layout_b8_; - c16->pb16i = pb16i_; - c16->t2s = t2s_; - // Which pb16i ELFs leave two int32 planes (the arena's ACC64 family: D, O and - // the full-attention q half). This mirrors the ELF family contract rather - // than inferring it, and load/launch refuse a tensor that needs two planes on - // a one-plane ELF. - if (pb16i_) { - const Q4nxDesign r = designs16_[i].role; - c16->pb16i_planes = - (r == Q4NX_DESIGN_D || r == Q4NX_DESIGN_O || r == Q4NX_DESIGN_QKVFA_Q) ? 2 : 1; - } - if (!c16->init_elf(*dev, elf.c_str(), designs16_[i].M, - designs16_[i].K, designs16_[i].N, 1, kname.c_str())) { - if (pb16i_) { - err_ = "pb16i: init_elf failed for " + elf; - return false; - } - fprintf(stderr, " [forward] M=16 twin failed to load (%s); " - "this role keeps M=128\n", designs16_[i].elf_name); - xd_->drop_ctx16(i); - skipped++; - continue; - } - made++; - fprintf(stderr, " [forward] %-7s M=%d K=%d N=%d %s (decode)\n", - role_tag(designs_[i].role), designs16_[i].M, designs16_[i].K, - designs16_[i].N, elf.c_str()); - } - xd_->note_m16_made(made > 0); + if (!xd_->open_decode_family(designs16_, ndesigns_, pb16i_sealed_, &made, &skipped, err_)) return false; if (pb16i_ && made < ndesigns_ - 0) { fprintf(stderr, " [forward] %s: %d of %d designs have a decode kernel\n", t2s_ ? "t2s" : "pb16i", made, ndesigns_); diff --git a/npu/target/xdna/xdna_target.cpp b/npu/target/xdna/xdna_target.cpp index 3c6dcdc45..b357fb3a0 100644 --- a/npu/target/xdna/xdna_target.cpp +++ b/npu/target/xdna/xdna_target.cpp @@ -541,6 +541,102 @@ void XdnaTarget::prof_dump() { prof_head_ = ProfAcc{}; } +namespace { +bool decode_elf_exists(const std::string& p) { + if (p.empty()) return false; + std::error_code ec; + const auto sz = std::filesystem::file_size(p, ec); + return !ec && sz > 0; +} +} // namespace + +bool XdnaTarget::open_decode_family(Q4nxDesignGeom* designs16, int n, const std::string& sealed_elf, + int* made, int* skipped, std::string& err) { + int made_n = 0, skipped_n = 0; + for (int i = 0; i < n; i++) { + if (designs16[i].M <= 0) { ++skipped_n; continue; } + std::string elf, kname = "main:seq"; + if (setup_.pb16i && !sealed_elf.empty()) { + // One ELF for every ternary phase: the file is the same for all roles and only the instance name + // differs, so I8Ctx reuses the one hw_context. + const char* inst = q4nx_pb16i_instance(designs16[i].role); + if (!inst) { + std::fprintf(stderr, + " [forward] pb16i: the sealed ELF has no instance for role %s; this role has no " + "decode kernel\n", + q4nx_design_log_tag(designs16[i].role)); + designs16[i].M = 0; + ++skipped_n; + continue; + } + elf = setup_.kernels_dir + "/" + sealed_elf; + kname = inst; + if (!decode_elf_exists(elf)) { + err = plan_.loaded() ? ("plan " + setup_.plan_path + ": decode.sealed_elf \"" + sealed_elf + + "\" is not in " + setup_.kernels_dir + " (missing)") + : ("pb16i: missing sealed ELF " + elf); + return false; + } + } else { + for (const std::string& d : setup_.decode_dirs) { + const std::string cand = d + "/" + designs16[i].elf_name; + if (decode_elf_exists(cand)) { elf = cand; break; } + } + } + if (elf.empty()) { + // A pb16i role without its image kernel must stop the load: it is not a slower path, it is a missing + // one (the int8 weights are not packed for these geometries under this layout). + if (setup_.pb16i) { + err = plan_.loaded() ? std::string("plan ") + setup_.plan_path + ": decode role " + + q4nx_design_log_tag(designs16[i].role) + " needs \"" + + designs16[i].elf_name + "\" (missing)" + : std::string("pb16i: missing decode ELF ") + designs16[i].elf_name; + return false; + } + ++skipped_n; + continue; // stay on M=128 + } + I8Ctx* c16 = make_ctx16(i, designs16[i]); + if (!c16) { + err = "cannot create the decode context for role " + std::to_string(i); + return false; + } + c16->MD = designs16[i].M; + c16->KD = designs16[i].K; + c16->ND = designs16[i].N; + c16->blocked_b = setup_.b8; + c16->pb16i = setup_.pb16i; + c16->t2s = setup_.t2s; + // Which pb16i ELFs leave two int32 planes (the arena's ACC64 family: D, O and the full-attention q half). + // This mirrors the ELF family contract rather than inferring it, and load/launch refuse a tensor that + // needs two planes on a one-plane ELF. + if (setup_.pb16i) { + const Q4nxDesign r = designs16[i].role; + c16->pb16i_planes = (r == Q4NX_DESIGN_D || r == Q4NX_DESIGN_O || r == Q4NX_DESIGN_QKVFA_Q) ? 2 : 1; + } + if (!c16->init_elf(*dev_, elf.c_str(), designs16[i].M, designs16[i].K, designs16[i].N, 1, kname.c_str())) { + if (setup_.pb16i) { + err = "pb16i: init_elf failed for " + elf; + return false; + } + std::fprintf(stderr, + " [forward] M=16 twin failed to load (%s); this role keeps M=128\n", + designs16[i].elf_name); + drop_ctx16(i); + ++skipped_n; + continue; + } + ++made_n; + std::fprintf(stderr, " [forward] %-7s M=%d K=%d N=%d %s (decode)\n", + q4nx_design_log_tag(designs16[i].role), designs16[i].M, designs16[i].K, designs16[i].N, + elf.c_str()); + } + note_m16_made(made_n > 0); + if (made) *made = made_n; + if (skipped) *skipped = skipped_n; + return true; +} + bool XdnaTarget::open_head(const Q4nxDesignGeom* designs, int n, std::string& err) { // run_lm_head packs int8 B tiles from the f32 head and launches them itself, so it // needs an int8 context with K=H. Under pb16i every decode context is a pb16i one, diff --git a/npu/target/xdna/xdna_target.h b/npu/target/xdna/xdna_target.h index 3ba8ae2ce..39d0ed60f 100644 --- a/npu/target/xdna/xdna_target.h +++ b/npu/target/xdna/xdna_target.h @@ -163,6 +163,17 @@ class XdnaTarget final : public NpuTarget { // forward and arrives through Setup, because deciding which sidecar file the head has is the container's // rule and not a target's. bool open_head(const Q4nxDesignGeom* designs, int n, std::string& err); + // The M=16 decode family (engine#803 A2b): for each role, resolve its `_m16` ELF under the plan's decode + // directories (or the sealed pb16i image, whose instance name comes from q4nx_pb16i_instance), create the + // context and configure it. `made`/`skipped` are what the forward still logs the summary from; a role whose + // ELF is absent keeps its M=128 kernel (a slower path, not a missing one) EXCEPT under pb16i, where the int8 + // weights are not packed for these geometries and the load must stop by name. This is the block that used to + // live in Q4nxNpuForward::init_impl -- here because it is ELF resolution and context creation, which is what a + // target owns; the container's naming (`sealed_elf`) still arrives from the forward. + // `designs16` is mutable: a role with no decode kernel has its M zeroed ("no decode design"), which is how the + // launch path knows to stay on M=128 -- the flag lives in the design, not in a parallel array. + bool open_decode_family(Q4nxDesignGeom* designs16, int n, const std::string& sealed_elf, int* made, + int* skipped, std::string& err); // Whether open_head() found and opened the pb16i ternary head. The model path reads this flag in four // places, so it stays the forward's; the target writes it. bool head_is_pb16i() const { return head_pb16i_; } From 6579423f21c0f786f7fce02f371edd1524e7cb61 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Mon, 9 Nov 2026 21:46:24 -0400 Subject: [PATCH 2/2] npu: delete the delegating stub -- the move's last call site is gone The review's note on 9caa9630 asked for the stub's lifetime to be stated, and the honest answer was (cherry picked from commit f444ee698208c26dada67fa7767655317547a51c) --- npu/forward/src/q4nx_forward.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index eb066050b..d2bd410d6 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -121,10 +121,6 @@ static const uint8_t* data_of(ModelWeights* mw, const TensorDesc* d, size_t byte return (const uint8_t*)model_tensor_data(mw, const_cast(d)); } -// Which instance of the sealed ELF runs a role, or nullptr when the sealed ELF does not cover it. The table -// moved to q4nx_pack.c (q4nx_pb16i_instance) because the target opens the decode family (engine#803 A2b) and a -// contract both sides must agree on cannot live on one side; this wrapper keeps the call sites unchanged. -static const char* pb16i_instance(Q4nxDesign r) { return q4nx_pb16i_instance(r); } // Where an M=16 int8 twin is looked for, first hit wins: /kernels-m16, // /m16, then itself.