diff --git a/npu/forward/include/q4nx_forward.h b/npu/forward/include/q4nx_forward.h index 22901baaf..9618e07f4 100644 --- a/npu/forward/include/q4nx_forward.h +++ b/npu/forward/include/q4nx_forward.h @@ -244,8 +244,8 @@ class Q4nxNpuForward { // Set only while step_batch_all() runs: the batched epilogue then norms every row and writes M x NV // logits there instead of only the last row's. Null on every other path, so nothing else changes. std::vector* batch_all_out_ = nullptr; - // engine#803 A2b step one: the decode-family block as its own method, body unchanged, so the move into - // XdnaTarget::open_decode_family() is a copy rather than surgery inside init_impl. + // engine#803 A2b: the model's half of the decode family (design names, summary lines, the wide and narrow + // sealed families); the per-role ELF resolution and context creation is XdnaTarget::open_decode_family(). bool open_decode_family_local(bool want_m16, xrt::device* dev, const std::string& mdir, const std::string& dir); diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index a5c56b8f1..645c5860f 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -430,14 +430,15 @@ bool Q4nxNpuForward::init_host(const char* model_path, const std::string& dir) { return init_impl(nullptr, model_path, dir); } -// engine#803 A2b, step one: the decode family gets its own method with the body unchanged, exactly as the lm -// head did before it moved behind XdnaTarget (open_head). That precedent is why this is a move and not a -// rewrite: the next step is then a copy into the target rather than surgery inside init_impl, and the three -// log lines below are copied byte-for-byte because gates grep them ("[pb16i-trace]", the decode-family -// line, the pb16i refusal) -- "improving" a message here breaks a gate that reads it. +// engine#803 A2b: the model's half of opening the decode family. It derives the decode designs and their ELF +// names (the container's naming), hands the per-role ELF resolution and context creation to +// XdnaTarget::open_decode_family(), prints the summary lines, and loads the wide and narrow sealed families on +// top. The log lines here and in the target are byte-for-byte what they were, because gates grep them (the +// decode-family line, the pb16i refusal) -- "improving" a message breaks a gate that reads it. bool Q4nxNpuForward::open_decode_family_local(bool want_m16, xrt::device* dev, const std::string& mdir, const std::string& dir) { if (host_gemm_) return true; // the host arm packs its own weights; no decode family is loaded + (void)mdir; // the decode directories reach the target through Setup::decode_dirs, resolved in init_impl if (!want_m16) { fprintf(stderr, " [forward] decode family: %d role(s) at M=%d, %d on M=%d%s\n", 0, Q4NX_DESIGN_M_DECODE, ndesigns_, Q4NX_DESIGN_M_PREFILL, " (NPU_DECODE_M16=0: disabled)"); @@ -500,100 +501,13 @@ bool Q4nxNpuForward::open_decode_family_local(bool want_m16, xrt::device* dev, c snprintf(designs16_[i].elf_name, Q4NX_ELF_NAME_MAX, "%s", n16.c_str()); } } - std::vector m16dirs; - plan_decode_dirs_.clear(); - if (plan_.loaded() && !plan_.decode_dirs.empty()) { - // The plan says which of the two given paths each candidate hangs off, so the runtime - // stops guessing which base a bare directory name meant. - for (const auto& d : plan_.decode_dirs) - plan_decode_dirs_.push_back((d.under_model ? mdir : dir) + "/" + d.path); - for (const auto& p : plan_decode_dirs_) m16dirs.push_back(p); - } else { - m16dirs.push_back(mdir + "/kernels-m16"); - m16dirs.push_back(dir + "/m16"); - m16dirs.push_back(dir); - } + // The per-role loop is the target's (engine#803 A2b): it resolves each role's `_m16` ELF under the decode + // directories the setup carried (the plan's, or the built-in three), or the sealed pb16i image, creates the + // context and configures it. What stays here is the container's naming -- the designs' ELF names above and + // which sidecar file the sealed ELF is -- the summary lines, and the wide and narrow families below. int made = 0, skipped = 0; - for (int i = 0; i < ndesigns_; i++) { - if (designs16_[i].M <= 0) { skipped++; continue; } - std::string elf, kname = "main:seq"; - if (pb16i_ && !pb16i_sealed_.empty()) { - // One ELF for every ternary phase: the file is the same for all roles and - // only the instance name differs, so I8Ctx reuses the one hw_context. - const char* inst = pb16i_instance(designs16_[i].role); - if (!inst) { - fprintf(stderr, " [forward] pb16i: the sealed ELF has no instance for " - "role %s; this role has no decode kernel\n", - role_tag(designs16_[i].role)); - designs16_[i].M = 0; - skipped++; - continue; - } - elf = dir + "/" + pb16i_sealed_; - sealed_dir_ = dir; - kname = inst; - if (!file_exists(elf)) { - err_ = plan_.loaded() ? ("plan " + plan_path_ + ": decode.sealed_elf \"" + - pb16i_sealed_ + "\" is not in " + dir + " (missing)") - : ("pb16i: missing sealed ELF " + elf); - return false; - } - } else { - for (const std::string& d : m16dirs) { - std::string cand = d + "/" + designs16_[i].elf_name; - if (file_exists(cand)) { elf = cand; break; } - } - } - if (elf.empty()) { - // A pb16i role without its image kernel must stop the load: it is not - // a slower path, it is a missing one (the int8 weights are not packed - // for these geometries under this layout). - if (pb16i_) { - err_ = plan_.loaded() - ? std::string("plan ") + plan_path_ + ": decode role " + - role_tag(designs16_[i].role) + " needs \"" + designs16_[i].elf_name + - "\" (missing)" - : std::string("pb16i: missing decode ELF ") + designs16_[i].elf_name; - return false; - } - skipped++; - continue; // stay on M=128 - } - I8Ctx* c16 = xd_->make_ctx16(i, designs16_[i]); - if (!c16) { err_ = "cannot create the decode context for role " + std::to_string(i); return false; } - c16->MD = designs16_[i].M; - c16->KD = designs16_[i].K; - c16->ND = designs16_[i].N; - c16->blocked_b = weight_layout_b8_; - c16->pb16i = pb16i_; - c16->t2s = t2s_; - // Which pb16i ELFs leave two int32 planes (the arena's ACC64 family: D, O and - // the full-attention q half). This mirrors the ELF family contract rather - // than inferring it, and load/launch refuse a tensor that needs two planes on - // a one-plane ELF. - if (pb16i_) { - const Q4nxDesign r = designs16_[i].role; - c16->pb16i_planes = - (r == Q4NX_DESIGN_D || r == Q4NX_DESIGN_O || r == Q4NX_DESIGN_QKVFA_Q) ? 2 : 1; - } - if (!c16->init_elf(*dev, elf.c_str(), designs16_[i].M, - designs16_[i].K, designs16_[i].N, 1, kname.c_str())) { - if (pb16i_) { - err_ = "pb16i: init_elf failed for " + elf; - return false; - } - fprintf(stderr, " [forward] M=16 twin failed to load (%s); " - "this role keeps M=128\n", designs16_[i].elf_name); - xd_->drop_ctx16(i); - skipped++; - continue; - } - made++; - fprintf(stderr, " [forward] %-7s M=%d K=%d N=%d %s (decode)\n", - role_tag(designs_[i].role), designs16_[i].M, designs16_[i].K, - designs16_[i].N, elf.c_str()); - } - xd_->note_m16_made(made > 0); + if (pb16i_ && !pb16i_sealed_.empty()) sealed_dir_ = dir; // where the wide/narrow sealed files are looked for + if (!xd_->open_decode_family(designs16_, ndesigns_, pb16i_sealed_, &made, &skipped, err_)) return false; decode_roles_ = made; // engine#2102: the caller-facing count, beside the byte-for-byte log line below if (pb16i_ && made < ndesigns_ - 0) { fprintf(stderr, " [forward] %s: %d of %d designs have a decode kernel\n", @@ -1472,6 +1386,11 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, setup.pb16i_dir = pb16i_dir_; setup.head_pb16i_images = pb16i_ && !pb16i_dir_.empty() && file_exists(head_pb16i_stem(0) + image_ext()); + // The plan's decode directories, resolved before the target is configured: the decode family and the + // lm-head both search setup.decode_dirs, so a list filled after this line reaches neither. (It used to be + // filled inside the M=16 block further down, after this point, so a plan's directories never reached the + // head lookup; with the decode loop on the target, nothing else would fill it at all.) + plan_decode_dirs_ = plan_.resolved_decode_dirs(mdir, dir); setup.decode_dirs = m16dirs_like_head(mdir, dir, plan_decode_dirs_); setup.hidden = c_.H; setup.vocab = c_.NV; @@ -1520,8 +1439,8 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, xd_->set_m16_check(getenv("NPU_M16_CHECK") != nullptr); const char* m16_env = getenv("NPU_DECODE_M16"); const bool want_m16 = !m16_env || atoi(m16_env) != 0; - // engine#803 A2b (step one): the block moved verbatim into open_decode_family_local(); the call site keeps the - // guard so the "disabled" log line is still printed from one place. + // engine#803 A2b: open_decode_family_local() names the decode designs and the target opens them; the call site + // keeps the guard so the "disabled" log line is still printed from one place. if (!open_decode_family_local(want_m16, dev, mdir, dir)) return false; // The lm head's own context (pb16i ternary images, the int8 ladder, or the q4_1 gemv head). Cut out diff --git a/npu/plan/plan.h b/npu/plan/plan.h index 7fcea0a09..4cefef8fb 100644 --- a/npu/plan/plan.h +++ b/npu/plan/plan.h @@ -205,6 +205,17 @@ struct Plan { // relative directory under it ("." = that directory itself). struct DecodeDir { bool under_model = false; std::string path; }; std::vector decode_dirs; + // decode_dirs as the paths to search, in the plan's order: each entry under the model directory or the + // kernel directory it names. Empty when the plan names none (or is not loaded), which is the caller's cue + // to fall back to its built-in places. One function, because the projections' decode twins and the lm-head's + // have to be looked for in the same list (engine#910), and the list was once built in a block that a move + // took away, leaving every caller on the built-in places with no error (engine#1984 review). + std::vector resolved_decode_dirs(const std::string& model_dir, const std::string& kernels_dir) const { + std::vector out; + if (!loaded()) return out; + for (const DecodeDir& d : decode_dirs) out.push_back((d.under_model ? model_dir : kernels_dir) + "/" + d.path); + return out; + } std::string sealed_elf; // pb16i: one sealed ELF, per-role instance names // engine#1990: an optional wide prefill family -- a second sealed ELF with the same instances built for // `sealed_wide_rows` rows per launch. Empty = none; the decode family then also runs prefill chunks. diff --git a/npu/target/xdna/xdna_target.cpp b/npu/target/xdna/xdna_target.cpp index fe1a67fcb..73afcf396 100644 --- a/npu/target/xdna/xdna_target.cpp +++ b/npu/target/xdna/xdna_target.cpp @@ -597,6 +597,103 @@ void XdnaTarget::prof_dump() { prof_head_ = ProfAcc{}; } +namespace { +// Same test the forward's file_exists() made: the file opens and is not empty. +bool decode_elf_exists(const std::string& p) { + FILE* f = std::fopen(p.c_str(), "rb"); + if (!f) return false; + std::fseek(f, 0, SEEK_END); + const long sz = std::ftell(f); + std::fclose(f); + return sz >= 1; +} +} // namespace + +// The log lines and refusals below are byte-for-byte the forward's: gates grep them. +bool XdnaTarget::open_decode_family(Q4nxDesignGeom* designs16, int n, const std::string& sealed_elf, + int* made, int* skipped, std::string& err) { + int made_n = 0, skipped_n = 0; + for (int i = 0; i < n; i++) { + if (designs16[i].M <= 0) { ++skipped_n; continue; } + std::string elf, kname = "main:seq"; + if (setup_.pb16i && !sealed_elf.empty()) { + // One ELF for every ternary phase: the file is the same for all roles and only the instance name + // differs, so I8Ctx reuses the one hw_context. + const char* inst = q4nx_pb16i_instance(designs16[i].role); + if (!inst) { + std::fprintf(stderr, " [forward] pb16i: the sealed ELF has no instance for " + "role %s; this role has no decode kernel\n", + q4nx_design_log_tag(designs16[i].role)); + designs16[i].M = 0; + ++skipped_n; + continue; + } + elf = setup_.kernels_dir + "/" + sealed_elf; + kname = inst; + if (!decode_elf_exists(elf)) { + err = plan_.loaded() ? ("plan " + setup_.plan_path + ": decode.sealed_elf \"" + sealed_elf + + "\" is not in " + setup_.kernels_dir + " (missing)") + : ("pb16i: missing sealed ELF " + elf); + return false; + } + } else { + for (const std::string& d : setup_.decode_dirs) { + const std::string cand = d + "/" + designs16[i].elf_name; + if (decode_elf_exists(cand)) { elf = cand; break; } + } + } + if (elf.empty()) { + // A pb16i role without its image kernel must stop the load: it is not a slower path, it is a missing + // one (the int8 weights are not packed for these geometries under this layout). + if (setup_.pb16i) { + err = plan_.loaded() ? std::string("plan ") + setup_.plan_path + ": decode role " + + q4nx_design_log_tag(designs16[i].role) + " needs \"" + designs16[i].elf_name + + "\" (missing)" + : std::string("pb16i: missing decode ELF ") + designs16[i].elf_name; + return false; + } + ++skipped_n; + continue; // stay on M=128 + } + I8Ctx* c16 = make_ctx16(i, designs16[i]); + if (!c16) { + err = "cannot create the decode context for role " + std::to_string(i); + return false; + } + c16->MD = designs16[i].M; + c16->KD = designs16[i].K; + c16->ND = designs16[i].N; + c16->blocked_b = setup_.b8; + c16->pb16i = setup_.pb16i; + c16->t2s = setup_.t2s; + // Which pb16i ELFs leave two int32 planes (the arena's ACC64 family: D, O and the full-attention q half). + // This mirrors the ELF family contract rather than inferring it, and load/launch refuse a tensor that + // needs two planes on a one-plane ELF. + if (setup_.pb16i) { + const Q4nxDesign r = designs16[i].role; + c16->pb16i_planes = (r == Q4NX_DESIGN_D || r == Q4NX_DESIGN_O || r == Q4NX_DESIGN_QKVFA_Q) ? 2 : 1; + } + if (!c16->init_elf(*dev_, elf.c_str(), designs16[i].M, designs16[i].K, designs16[i].N, 1, kname.c_str())) { + if (setup_.pb16i) { + err = "pb16i: init_elf failed for " + elf; + return false; + } + std::fprintf(stderr, " [forward] M=16 twin failed to load (%s); " + "this role keeps M=128\n", designs16[i].elf_name); + drop_ctx16(i); + ++skipped_n; + continue; + } + ++made_n; + std::fprintf(stderr, " [forward] %-7s M=%d K=%d N=%d %s (decode)\n", + q4nx_design_log_tag(designs16[i].role), designs16[i].M, designs16[i].K, designs16[i].N, elf.c_str()); + } + note_m16_made(made_n > 0); + if (made) *made = made_n; + if (skipped) *skipped = skipped_n; + return true; +} + bool XdnaTarget::open_head(const Q4nxDesignGeom* designs, int n, std::string& err) { // run_lm_head packs int8 B tiles from the f32 head and launches them itself, so it // needs an int8 context with K=H. Under pb16i every decode context is a pb16i one, diff --git a/npu/target/xdna/xdna_target.h b/npu/target/xdna/xdna_target.h index 261f6c197..cc3767933 100644 --- a/npu/target/xdna/xdna_target.h +++ b/npu/target/xdna/xdna_target.h @@ -163,6 +163,18 @@ class XdnaTarget final : public NpuTarget { // forward and arrives through Setup, because deciding which sidecar file the head has is the container's // rule and not a target's. bool open_head(const Q4nxDesignGeom* designs, int n, std::string& err); + // The M=16 decode family (engine#803 A2b): for each role, resolve its `_m16` ELF under Setup::decode_dirs (or + // the sealed pb16i image in the kernels directory, whose instance name comes from q4nx_pb16i_instance), create + // the context and configure it. `made`/`skipped` are what the forward logs its summary line from. A role + // whose ELF is absent keeps its M=128 kernel (a slower path, not a missing one) EXCEPT under pb16i, where the + // int8 weights are not packed for these geometries and the load stops by name. This is the per-role loop that + // lived in Q4nxNpuForward: it is ELF resolution and context creation, which a target owns. The container's + // naming (`sealed_elf`, and the designs' elf_name) still arrives from the forward, and so do the wide and + // narrow sealed families, which the forward loads on top of the contexts this opens. + // `designs16` is mutable: a role with no decode kernel has its M zeroed, which is how the launch path knows + // to stay on M=128 -- the flag lives in the design, not in a parallel array. + bool open_decode_family(Q4nxDesignGeom* designs16, int n, const std::string& sealed_elf, int* made, + int* skipped, std::string& err); // Whether open_head() found and opened the pb16i ternary head. The model path reads this flag in four // places, so it stays the forward's; the target writes it. bool head_is_pb16i() const { return head_pb16i_; } diff --git a/tests/npu_plan_test.cpp b/tests/npu_plan_test.cpp index c1d0992c1..ab4e67e9d 100644 --- a/tests/npu_plan_test.cpp +++ b/tests/npu_plan_test.cpp @@ -213,6 +213,21 @@ void test_valid() { "the pb16i family is read with its sealed ELF"); check(pp.decode_dirs.size() == 2 && pp.decode_dirs[0].under_model, "a model-relative decode dir is kept"); + // The search list a target is configured with: the plan's entries, each under the directory it names, in the + // plan's order (engine#1984 review: nothing resolved them, so a plan naming any place but the built-in three + // lost its decode twins without an error). + { + const std::vector got = pp.resolved_decode_dirs("/m", "/k"); + check(got.size() == 2 && got[0] == "/m/kernels-m16" && got[1] == "/k/.", + "decode dirs resolve under the model dir or the kernel dir, in the plan's order"); + auto seq = base_plan(); + seq["decode"]["dirs"] = nlohmann::json::array({nlohmann::json{{"root", "model"}, {"path", "kernels-m16-seq"}}}); + const std::vector only = Plan::load(write_plan("seq", seq)).resolved_decode_dirs("/m", "/k"); + check(only.size() == 1 && only[0] == "/m/kernels-m16-seq", + "a plan naming one directory outside the built-in places resolves to exactly that directory"); + check(Plan().resolved_decode_dirs("/m", "/k").empty(), "a plan that is not loaded resolves to no directories"); + } + // attention.mode is optional and defaults to causal, so a plan written before the field loads and // still means what it meant then. check(p.attention_mode == "causal" && !p.noncausal(), "a plan with no attention block is causal");