diff --git a/npu/forward/include/q4nx_pack.h b/npu/forward/include/q4nx_pack.h index 922caa07f..6621d90e5 100644 --- a/npu/forward/include/q4nx_pack.h +++ b/npu/forward/include/q4nx_pack.h @@ -246,6 +246,15 @@ int q4nx_derive_decode_designs(const Q4nxDesignGeom* designs, int count, * is a key (engine#803 A2), so it uses these; the log keeps the short tag. */ const char* q4nx_design_role_name(Q4nxDesign role); +/* The LOG tag for a design role -- deliberately NOT the unique name above, and deliberately the + * historical mapping (QKVLIN prints as "QKV", every role the old table did not name prints as "D"). + * + * It exists so a diagnostic a reader has already learned does not change when code moves: the setup + * side moved into the target (engine#803 A2b) and the log lines had to stay byte-identical, which the + * #1380 review caught them not doing. The mapping is arguably misleading for the MLA and MoE roles -- + * that is a change worth making on purpose, in a PR that says so, not as a side effect of a move. */ +const char* q4nx_design_log_tag(Q4nxDesign role); + /* Look up the entry for a role in a derived set; NULL if absent. */ const Q4nxDesignGeom* q4nx_find_design(const Q4nxDesignGeom* designs, int count, Q4nxDesign role); diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 205d1953a..a523e9c46 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -391,26 +391,9 @@ int Q4nxNpuForward::idx_of(Q4nxDesign role) const { return -1; } -static const char* role_tag(Q4nxDesign r) { - switch (r) { - case Q4NX_DESIGN_QKV: return "QKV"; - case Q4NX_DESIGN_QKV2: return "QKV2"; - case Q4NX_DESIGN_O: return "O"; - case Q4NX_DESIGN_O2: return "O2"; - case Q4NX_DESIGN_GU: return "GU"; - case Q4NX_DESIGN_G: return "G"; - case Q4NX_DESIGN_U: return "U"; - case Q4NX_DESIGN_PLEGATE: return "PLEGATE"; - case Q4NX_DESIGN_PLEPROJ: return "PLEPROJ"; - case Q4NX_DESIGN_GU2: return "GU2"; - case Q4NX_DESIGN_D2: return "D2"; - case Q4NX_DESIGN_QKVLIN: return "QKV"; - case Q4NX_DESIGN_Z: return "Z"; - case Q4NX_DESIGN_QKVFA_Q: return "QKVFAQ"; - case Q4NX_DESIGN_QKVFA_KV: return "QKVFAKV"; - default: return "D"; - } -} +// The log tag lives with the geometry it describes (q4nx_pack.c) so the target's own setup messages and +// this file's can never drift apart; it is the historical mapping, not the unique plan name. +static const char* role_tag(Q4nxDesign r) { return q4nx_design_log_tag(r); } // -------------------------------------------------------------------------- // init @@ -431,7 +414,7 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, // The launch layer. A1: everything below that touches a device talks to this, and the settings the // container declares (weight layout, pb16i image dir, Hadamard fold) are pushed into it as they are // parsed. A2 turns this into NpuTarget::open(plan, kernels_dir). - xd_ = std::make_unique(); + xd_ = std::make_unique(dev); int had_block = 1024; bool weight_cache = true; if (getenv("NPU_NO_WCACHE")) weight_cache = false; // A/B: disable the packed-weight cache @@ -1116,46 +1099,27 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, } } - for (int i = 0; i < ndesigns_; i++) { - const Q4nxDesignGeom* g = &designs_[i]; - // The two full-attention-QKV-split designs exist only as pb16i kernels (the - // role has no int8 launch that fits), so they get no M=128 context and no - // int8 ELF on disk: their decode context is built below from the _pb16i ELF. - xd_->adopt_geometry(i, *g); // the target owns the geometry it launches on - if (xd_->role_is_pb16i_split(i)) continue; // no int8 launch for these two roles - - if (host_gemm_) { - // Offline mode: host GEMMs are plain float matmuls over the decoded weights, so - // no ELF is opened and none is required. A model whose kernel set has not been - // built yet can be loaded and prompted, which is what makes it a usable - // numeric reference for a new geometry (engine #796: the d3-lite backbone - // answers on the host while its six designs do not exist). The artifact is - // named when it is there; the ELF set itself is checked by - // tests/verify_design_elfs + tests/verify_full_elfs.sh, not by this path. - std::string elf = dir + "/" + g->elf_name; - if (file_exists(elf)) { - fprintf(stderr, " [forward] %-3s M=%d K=%d N=%d %s (host-GEMM)\n", - role_tag(g->role), g->M, g->K, g->N, elf.c_str()); - } else { - fprintf(stderr, " [forward] %-3s M=%d K=%d N=%d (host-GEMM, no ELF: %s)\n", - role_tag(g->role), g->M, g->K, g->N, g->elf_name); - } - } else { - std::string elf = dir + "/" + g->elf_name; - if (!file_exists(elf)) { - err_ = missing_kernel(dir, g->elf_name, g->role); - return false; - } - I8Ctx* ctx = xd_->make_ctx(i); // adopted: the target owns the context from here - if (!ctx) { err_ = "cannot create a context for role " + std::to_string(i); return false; } - ctx->MD = g->M; ctx->KD = g->K; ctx->ND = g->N; - ctx->blocked_b = weight_layout_b8_; - if (!ctx->init_elf(*dev, elf.c_str(), g->M, g->K, g->N, 1)) { - err_ = "I8Ctx init_elf failed for " + elf; - return false; - } - fprintf(stderr, " [forward] %-3s M=%d K=%d N=%d %s\n", role_tag(g->role), - g->M, g->K, g->N, elf.c_str()); + // The M=128 ELF set is the target's to open (engine#803 A2b): it adopts the geometry, probes the + // artifact and refuses by name, and the forward no longer touches an I8Ctx to do it. + { + // What the setup side reads, set once: the remaining two blocks (decode family, lm-head) read + // the same fields, so moving them later needs no new plumbing (engine#803 A2b). + npu::target::xdna::XdnaTarget::Setup setup; + setup.model_dir = mdir; + setup.kernels_dir = dir; + setup.family = family_tag(); + setup.plan_path = plan_.loaded() ? plan_path_ : std::string(); + setup.b8 = weight_layout_b8_; + setup.pb16i = pb16i_; + setup.t2s = t2s_; + setup.t2s_v2 = t2s_v2_; + setup.host_gemm = host_gemm_; + setup.plan = plan_.loaded() ? &plan_ : nullptr; + xd_->configure(setup); + std::string why; + if (!xd_->open_prefill(designs_, ndesigns_, why)) { + err_ = std::move(why); + return false; } } diff --git a/npu/forward/src/q4nx_pack.c b/npu/forward/src/q4nx_pack.c index 2591eaa80..8354b5dfd 100644 --- a/npu/forward/src/q4nx_pack.c +++ b/npu/forward/src/q4nx_pack.c @@ -158,6 +158,27 @@ void q4nx_host_gemm_f32_padded(const float* A, int a_k, int M, int K, int N, con free(pad); } +const char* q4nx_design_log_tag(Q4nxDesign role) { + switch (role) { + case Q4NX_DESIGN_QKV: return "QKV"; + case Q4NX_DESIGN_QKV2: return "QKV2"; + case Q4NX_DESIGN_O: return "O"; + case Q4NX_DESIGN_O2: return "O2"; + case Q4NX_DESIGN_GU: return "GU"; + case Q4NX_DESIGN_G: return "G"; + case Q4NX_DESIGN_U: return "U"; + case Q4NX_DESIGN_PLEGATE: return "PLEGATE"; + case Q4NX_DESIGN_PLEPROJ: return "PLEPROJ"; + case Q4NX_DESIGN_GU2: return "GU2"; + case Q4NX_DESIGN_D2: return "D2"; + case Q4NX_DESIGN_QKVLIN: return "QKV"; + case Q4NX_DESIGN_Z: return "Z"; + case Q4NX_DESIGN_QKVFA_Q: return "QKVFAQ"; + case Q4NX_DESIGN_QKVFA_KV: return "QKVFAKV"; + default: return "D"; + } +} + const char* q4nx_design_role_name(Q4nxDesign role) { switch (role) { case Q4NX_DESIGN_QKV: return "QKV"; diff --git a/npu/target/xdna/xdna_target.cpp b/npu/target/xdna/xdna_target.cpp index 4ca6dae6c..2f92401c5 100644 --- a/npu/target/xdna/xdna_target.cpp +++ b/npu/target/xdna/xdna_target.cpp @@ -77,9 +77,85 @@ static const char* role_tag(Q4nxDesign r) { } // namespace -XdnaTarget::XdnaTarget() = default; +// The forward's own probe, kept identical (exists and non-empty): an ELF that is present but zero bytes +// is the failure this catches, and a target that decided differently from its caller would be a second +// definition of "the kernel is there". +static bool elf_present(const std::string& p) { + FILE* f = std::fopen(p.c_str(), "rb"); + if (!f) return false; + std::fseek(f, 0, SEEK_END); + const long sz = std::ftell(f); + std::fclose(f); + return sz >= 1; +} + +XdnaTarget::XdnaTarget(xrt::device* dev) : dev_(dev) {} XdnaTarget::~XdnaTarget() = default; +// The M=128 ELF set, moved out of Q4nxNpuForward::init_impl() unchanged (engine#803 A2b): the same +// probe, the same two refusals, the same per-role log line, and the same adopt-then-open order. What +// changed is who owns it -- a missing or unloadable artifact is now refused by the target that would +// launch it, instead of by a caller that then hands it a context. +bool XdnaTarget::open_prefill(const Q4nxDesignGeom* designs, int n, std::string& err) { + const std::string& dir = setup_.kernels_dir; + const bool b8 = setup_.b8; + const bool host_gemm = setup_.host_gemm; + const std::string& plan_path = setup_.plan_path; + if (!designs || n <= 0 || n > Q4NX_MAX_DESIGNS) { + err = "open_prefill: no design set (" + std::to_string(n) + " designs)"; + return false; + } + for (int i = 0; i < n; i++) { + const Q4nxDesignGeom* g = &designs[i]; + // The two full-attention-QKV-split designs exist only as pb16i kernels (the role has no int8 + // launch that fits), so they get no M=128 context and no int8 ELF on disk: their decode context + // is built from the _pb16i ELF by the caller's twin pass. + adopt_geometry(i, *g); + if (role_is_pb16i_split(i)) continue; + const std::string elf = dir + "/" + g->elf_name; + if (host_gemm) { + // Offline mode: host GEMMs are plain float matmuls over the decoded weights, so no ELF is + // opened and none is required. A model whose kernel set is not built yet can still be + // loaded and prompted, which is what makes it a numeric reference for a new geometry + // (engine#796). + if (elf_present(elf)) + std::fprintf(stderr, " [forward] %-3s M=%d K=%d N=%d %s (host-GEMM)\n", + q4nx_design_log_tag(g->role), g->M, g->K, g->N, elf.c_str()); + else + std::fprintf(stderr, " [forward] %-3s M=%d K=%d N=%d (host-GEMM, no ELF: %s)\n", + q4nx_design_log_tag(g->role), g->M, g->K, g->N, g->elf_name); + continue; + } + if (!elf_present(elf)) { + // The same wording the forward used for this case, because a diagnostic that changes shape + // when code moves is a diagnostic someone has to re-learn. + const char* role = q4nx_design_log_tag(g->role); + err = plan_path.empty() ? ("missing full ELF " + elf) + : ("plan " + plan_path + ": this model's plan needs the " + role + + " kernel \"" + g->elf_name + "\" in " + dir + " (missing)"); + return false; + } + if (!dev_) { + err = "open_prefill: no device for the ELF set"; + return false; + } + I8Ctx* ctx = make_ctx(i); + if (!ctx) { + err = "cannot create a context for role " + std::to_string(i); + return false; + } + ctx->MD = g->M; ctx->KD = g->K; ctx->ND = g->N; + ctx->blocked_b = b8; + if (!ctx->init_elf(*dev_, elf.c_str(), g->M, g->K, g->N, 1)) { + err = "I8Ctx init_elf failed for " + elf; + return false; + } + std::fprintf(stderr, " [forward] %-3s M=%d K=%d N=%d %s\n", q4nx_design_log_tag(g->role), + g->M, g->K, g->N, elf.c_str()); + } + return true; +} + // -------------------------------------------------------------------------- // NpuTarget // -------------------------------------------------------------------------- diff --git a/npu/target/xdna/xdna_target.h b/npu/target/xdna/xdna_target.h index c54aba277..ab3ed4583 100644 --- a/npu/target/xdna/xdna_target.h +++ b/npu/target/xdna/xdna_target.h @@ -59,7 +59,10 @@ struct ProfAcc { class XdnaTarget final : public NpuTarget { public: - XdnaTarget(); + // The device is the XDNA target's business, not the interface's: passing it here is what keeps + // NpuTarget::open(plan, kernels_dir, family, err) exactly as documented for a second target + // (engine#803 A2b). Null is legal and means "host GEMMs only" (init_host()). + explicit XdnaTarget(xrt::device* dev = nullptr); ~XdnaTarget() override; XdnaTarget(const XdnaTarget&) = delete; XdnaTarget& operator=(const XdnaTarget&) = delete; @@ -110,6 +113,42 @@ class XdnaTarget final : public NpuTarget { int max_rows() const; // widest launch this target will accept (decode twins included) int rows_for(int role, int want) const; // rows the phase-selected kernel for `role` can take + // Everything the setup side reads, set once before open_*(). It exists because the remaining two + // blocks (the M=16 decode family and the lm-head context) read a dozen of the forward's fields -- + // the plan, its path, the two given directories, the layout family, the sealed ELF, the t2s flag -- + // and a twelve-argument call would be a signature nobody can review. Once these live here, moving + // those blocks changes no call site (engine#803 A2b). + struct Setup { + std::string model_dir; // the -m directory + std::string kernels_dir; // the --kernels directory + std::string family; // "", "_b8", "_pb16i", "_t2s", "_t2s2" + std::string plan_path; // the plan this set came from, when one did ("" = derived) + bool b8 = false; // blocked 8x8 tile layout + bool pb16i = false; // integer-exact ternary, sidecar images + bool t2s = false; // 2-bit ternary stream + bool t2s_v2 = false; + bool host_gemm = false; // offline: open no ELF at all + const onebit::npu::plan::Plan* plan = nullptr; // copied into plan_ by configure(); null = none + }; + // Setup carries a pointer to the forward's Plan; this copies it, so from here on the target's own + // plan_ is THE copy the setup side reads (open() no longer has to be the one that filled it). Two + // live copies of the plan is the drift the plan layer exists to prevent, so there is one. + void configure(const Setup& s) { + setup_ = s; + if (s.plan) plan_ = *s.plan; + } + const Setup& setup() const { return setup_; } + + // The M=128 ELF set: for each design, adopt its geometry, probe its artifact and open it, refusing by + // name when an artifact is missing or will not load (engine#803 A2b). This is the block that used to + // live at the top of Q4nxNpuForward::init_impl(); it is here because it is ELF loading and its + // refusals, which is what a target owns. `host_gemm` opens nothing (the offline path needs no ELF) + // and only names the artifacts it would have used. `b8` is the container's blocked-tile layout. + // `plan_path` is the plan the set came from, when one did: the missing-artifact refusal quotes it, so + // the message a user sees is the same one the forward produced before this moved (an empty string + // means no plan, i.e. a purely derived set). + bool open_prefill(const Q4nxDesignGeom* designs, int n, std::string& err); + // Context construction: the caller configures and initialises the I8Ctx, then it stays here. I8Ctx* make_ctx(int role); I8Ctx* ctx(int role) const; @@ -150,6 +189,7 @@ class XdnaTarget final : public NpuTarget { std::string pb16i_stem(int layer, const char* tag) const; xrt::device* dev_ = nullptr; + Setup setup_; // see configure(): what the setup side reads, in one place int n_layers_ = 0; std::string kernels_dir_; onebit::npu::plan::Plan plan_; // when loaded: what open() checked this target against