Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions npu/forward/include/q4nx_forward.h
Original file line number Diff line number Diff line change
Expand Up @@ -244,8 +244,8 @@ class Q4nxNpuForward {
// Set only while step_batch_all() runs: the batched epilogue then norms every row and writes M x NV
// logits there instead of only the last row's. Null on every other path, so nothing else changes.
std::vector<float>* batch_all_out_ = nullptr;
// engine#803 A2b step one: the decode-family block as its own method, body unchanged, so the move into
// XdnaTarget::open_decode_family() is a copy rather than surgery inside init_impl.
// engine#803 A2b: the model's half of the decode family (design names, summary lines, the wide and narrow
// sealed families); the per-role ELF resolution and context creation is XdnaTarget::open_decode_family().
bool open_decode_family_local(bool want_m16, xrt::device* dev, const std::string& mdir,
const std::string& dir);

Expand Down
119 changes: 19 additions & 100 deletions npu/forward/src/q4nx_forward.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -430,14 +430,15 @@ bool Q4nxNpuForward::init_host(const char* model_path, const std::string& dir) {
return init_impl(nullptr, model_path, dir);
}

// engine#803 A2b, step one: the decode family gets its own method with the body unchanged, exactly as the lm
// head did before it moved behind XdnaTarget (open_head). That precedent is why this is a move and not a
// rewrite: the next step is then a copy into the target rather than surgery inside init_impl, and the three
// log lines below are copied byte-for-byte because gates grep them ("[pb16i-trace]", the decode-family
// line, the pb16i refusal) -- "improving" a message here breaks a gate that reads it.
// engine#803 A2b: the model's half of opening the decode family. It derives the decode designs and their ELF
// names (the container's naming), hands the per-role ELF resolution and context creation to
// XdnaTarget::open_decode_family(), prints the summary lines, and loads the wide and narrow sealed families on
// top. The log lines here and in the target are byte-for-byte what they were, because gates grep them (the
// decode-family line, the pb16i refusal) -- "improving" a message breaks a gate that reads it.
bool Q4nxNpuForward::open_decode_family_local(bool want_m16, xrt::device* dev, const std::string& mdir,
const std::string& dir) {
if (host_gemm_) return true; // the host arm packs its own weights; no decode family is loaded
(void)mdir; // the decode directories reach the target through Setup::decode_dirs, resolved in init_impl
if (!want_m16) {
fprintf(stderr, " [forward] decode family: %d role(s) at M=%d, %d on M=%d%s\n", 0,
Q4NX_DESIGN_M_DECODE, ndesigns_, Q4NX_DESIGN_M_PREFILL, " (NPU_DECODE_M16=0: disabled)");
Expand Down Expand Up @@ -500,100 +501,13 @@ bool Q4nxNpuForward::open_decode_family_local(bool want_m16, xrt::device* dev, c
snprintf(designs16_[i].elf_name, Q4NX_ELF_NAME_MAX, "%s", n16.c_str());
}
}
std::vector<std::string> m16dirs;
plan_decode_dirs_.clear();
if (plan_.loaded() && !plan_.decode_dirs.empty()) {
// The plan says which of the two given paths each candidate hangs off, so the runtime
// stops guessing which base a bare directory name meant.
for (const auto& d : plan_.decode_dirs)
plan_decode_dirs_.push_back((d.under_model ? mdir : dir) + "/" + d.path);
for (const auto& p : plan_decode_dirs_) m16dirs.push_back(p);
} else {
m16dirs.push_back(mdir + "/kernels-m16");
m16dirs.push_back(dir + "/m16");
m16dirs.push_back(dir);
}
// The per-role loop is the target's (engine#803 A2b): it resolves each role's `_m16` ELF under the decode
// directories the setup carried (the plan's, or the built-in three), or the sealed pb16i image, creates the
// context and configures it. What stays here is the container's naming -- the designs' ELF names above and
// which sidecar file the sealed ELF is -- the summary lines, and the wide and narrow families below.
int made = 0, skipped = 0;
for (int i = 0; i < ndesigns_; i++) {
if (designs16_[i].M <= 0) { skipped++; continue; }
std::string elf, kname = "main:seq";
if (pb16i_ && !pb16i_sealed_.empty()) {
// One ELF for every ternary phase: the file is the same for all roles and
// only the instance name differs, so I8Ctx reuses the one hw_context.
const char* inst = pb16i_instance(designs16_[i].role);
if (!inst) {
fprintf(stderr, " [forward] pb16i: the sealed ELF has no instance for "
"role %s; this role has no decode kernel\n",
role_tag(designs16_[i].role));
designs16_[i].M = 0;
skipped++;
continue;
}
elf = dir + "/" + pb16i_sealed_;
sealed_dir_ = dir;
kname = inst;
if (!file_exists(elf)) {
err_ = plan_.loaded() ? ("plan " + plan_path_ + ": decode.sealed_elf \"" +
pb16i_sealed_ + "\" is not in " + dir + " (missing)")
: ("pb16i: missing sealed ELF " + elf);
return false;
}
} else {
for (const std::string& d : m16dirs) {
std::string cand = d + "/" + designs16_[i].elf_name;
if (file_exists(cand)) { elf = cand; break; }
}
}
if (elf.empty()) {
// A pb16i role without its image kernel must stop the load: it is not
// a slower path, it is a missing one (the int8 weights are not packed
// for these geometries under this layout).
if (pb16i_) {
err_ = plan_.loaded()
? std::string("plan ") + plan_path_ + ": decode role " +
role_tag(designs16_[i].role) + " needs \"" + designs16_[i].elf_name +
"\" (missing)"
: std::string("pb16i: missing decode ELF ") + designs16_[i].elf_name;
return false;
}
skipped++;
continue; // stay on M=128
}
I8Ctx* c16 = xd_->make_ctx16(i, designs16_[i]);
if (!c16) { err_ = "cannot create the decode context for role " + std::to_string(i); return false; }
c16->MD = designs16_[i].M;
c16->KD = designs16_[i].K;
c16->ND = designs16_[i].N;
c16->blocked_b = weight_layout_b8_;
c16->pb16i = pb16i_;
c16->t2s = t2s_;
// Which pb16i ELFs leave two int32 planes (the arena's ACC64 family: D, O and
// the full-attention q half). This mirrors the ELF family contract rather
// than inferring it, and load/launch refuse a tensor that needs two planes on
// a one-plane ELF.
if (pb16i_) {
const Q4nxDesign r = designs16_[i].role;
c16->pb16i_planes =
(r == Q4NX_DESIGN_D || r == Q4NX_DESIGN_O || r == Q4NX_DESIGN_QKVFA_Q) ? 2 : 1;
}
if (!c16->init_elf(*dev, elf.c_str(), designs16_[i].M,
designs16_[i].K, designs16_[i].N, 1, kname.c_str())) {
if (pb16i_) {
err_ = "pb16i: init_elf failed for " + elf;
return false;
}
fprintf(stderr, " [forward] M=16 twin failed to load (%s); "
"this role keeps M=128\n", designs16_[i].elf_name);
xd_->drop_ctx16(i);
skipped++;
continue;
}
made++;
fprintf(stderr, " [forward] %-7s M=%d K=%d N=%d %s (decode)\n",
role_tag(designs_[i].role), designs16_[i].M, designs16_[i].K,
designs16_[i].N, elf.c_str());
}
xd_->note_m16_made(made > 0);
if (pb16i_ && !pb16i_sealed_.empty()) sealed_dir_ = dir; // where the wide/narrow sealed files are looked for
if (!xd_->open_decode_family(designs16_, ndesigns_, pb16i_sealed_, &made, &skipped, err_)) return false;
decode_roles_ = made; // engine#2102: the caller-facing count, beside the byte-for-byte log line below
if (pb16i_ && made < ndesigns_ - 0) {
fprintf(stderr, " [forward] %s: %d of %d designs have a decode kernel\n",
Expand Down Expand Up @@ -1472,6 +1386,11 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path,
setup.pb16i_dir = pb16i_dir_;
setup.head_pb16i_images =
pb16i_ && !pb16i_dir_.empty() && file_exists(head_pb16i_stem(0) + image_ext());
// The plan's decode directories, resolved before the target is configured: the decode family and the
// lm-head both search setup.decode_dirs, so a list filled after this line reaches neither. (It used to be
// filled inside the M=16 block further down, after this point, so a plan's directories never reached the
// head lookup; with the decode loop on the target, nothing else would fill it at all.)
plan_decode_dirs_ = plan_.resolved_decode_dirs(mdir, dir);
setup.decode_dirs = m16dirs_like_head(mdir, dir, plan_decode_dirs_);
setup.hidden = c_.H;
setup.vocab = c_.NV;
Expand Down Expand Up @@ -1520,8 +1439,8 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path,
xd_->set_m16_check(getenv("NPU_M16_CHECK") != nullptr);
const char* m16_env = getenv("NPU_DECODE_M16");
const bool want_m16 = !m16_env || atoi(m16_env) != 0;
// engine#803 A2b (step one): the block moved verbatim into open_decode_family_local(); the call site keeps the
// guard so the "disabled" log line is still printed from one place.
// engine#803 A2b: open_decode_family_local() names the decode designs and the target opens them; the call site
// keeps the guard so the "disabled" log line is still printed from one place.
if (!open_decode_family_local(want_m16, dev, mdir, dir)) return false;

// The lm head's own context (pb16i ternary images, the int8 ladder, or the q4_1 gemv head). Cut out
Expand Down
11 changes: 11 additions & 0 deletions npu/plan/plan.h
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,17 @@ struct Plan {
// relative directory under it ("." = that directory itself).
struct DecodeDir { bool under_model = false; std::string path; };
std::vector<DecodeDir> decode_dirs;
// decode_dirs as the paths to search, in the plan's order: each entry under the model directory or the
// kernel directory it names. Empty when the plan names none (or is not loaded), which is the caller's cue
// to fall back to its built-in places. One function, because the projections' decode twins and the lm-head's
// have to be looked for in the same list (engine#910), and the list was once built in a block that a move
// took away, leaving every caller on the built-in places with no error (engine#1984 review).
std::vector<std::string> resolved_decode_dirs(const std::string& model_dir, const std::string& kernels_dir) const {
std::vector<std::string> out;
if (!loaded()) return out;
for (const DecodeDir& d : decode_dirs) out.push_back((d.under_model ? model_dir : kernels_dir) + "/" + d.path);
return out;
}
std::string sealed_elf; // pb16i: one sealed ELF, per-role instance names
// engine#1990: an optional wide prefill family -- a second sealed ELF with the same instances built for
// `sealed_wide_rows` rows per launch. Empty = none; the decode family then also runs prefill chunks.
Expand Down
97 changes: 97 additions & 0 deletions npu/target/xdna/xdna_target.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -597,6 +597,103 @@ void XdnaTarget::prof_dump() {
prof_head_ = ProfAcc{};
}

namespace {
// Same test the forward's file_exists() made: the file opens and is not empty.
bool decode_elf_exists(const std::string& p) {
FILE* f = std::fopen(p.c_str(), "rb");
if (!f) return false;
std::fseek(f, 0, SEEK_END);
const long sz = std::ftell(f);
std::fclose(f);
return sz >= 1;
}
} // namespace

// The log lines and refusals below are byte-for-byte the forward's: gates grep them.
bool XdnaTarget::open_decode_family(Q4nxDesignGeom* designs16, int n, const std::string& sealed_elf,
int* made, int* skipped, std::string& err) {
int made_n = 0, skipped_n = 0;
for (int i = 0; i < n; i++) {
if (designs16[i].M <= 0) { ++skipped_n; continue; }
std::string elf, kname = "main:seq";
if (setup_.pb16i && !sealed_elf.empty()) {
// One ELF for every ternary phase: the file is the same for all roles and only the instance name
// differs, so I8Ctx reuses the one hw_context.
const char* inst = q4nx_pb16i_instance(designs16[i].role);
if (!inst) {
std::fprintf(stderr, " [forward] pb16i: the sealed ELF has no instance for "
"role %s; this role has no decode kernel\n",
q4nx_design_log_tag(designs16[i].role));
designs16[i].M = 0;
++skipped_n;
continue;
}
elf = setup_.kernels_dir + "/" + sealed_elf;
kname = inst;
if (!decode_elf_exists(elf)) {
err = plan_.loaded() ? ("plan " + setup_.plan_path + ": decode.sealed_elf \"" + sealed_elf +
"\" is not in " + setup_.kernels_dir + " (missing)")
: ("pb16i: missing sealed ELF " + elf);
return false;
}
} else {
for (const std::string& d : setup_.decode_dirs) {
const std::string cand = d + "/" + designs16[i].elf_name;
if (decode_elf_exists(cand)) { elf = cand; break; }
}
}
if (elf.empty()) {
// A pb16i role without its image kernel must stop the load: it is not a slower path, it is a missing
// one (the int8 weights are not packed for these geometries under this layout).
if (setup_.pb16i) {
err = plan_.loaded() ? std::string("plan ") + setup_.plan_path + ": decode role " +
q4nx_design_log_tag(designs16[i].role) + " needs \"" + designs16[i].elf_name +
"\" (missing)"
: std::string("pb16i: missing decode ELF ") + designs16[i].elf_name;
return false;
}
++skipped_n;
continue; // stay on M=128
}
I8Ctx* c16 = make_ctx16(i, designs16[i]);
if (!c16) {
err = "cannot create the decode context for role " + std::to_string(i);
return false;
}
c16->MD = designs16[i].M;
c16->KD = designs16[i].K;
c16->ND = designs16[i].N;
c16->blocked_b = setup_.b8;
c16->pb16i = setup_.pb16i;
c16->t2s = setup_.t2s;
// Which pb16i ELFs leave two int32 planes (the arena's ACC64 family: D, O and the full-attention q half).
// This mirrors the ELF family contract rather than inferring it, and load/launch refuse a tensor that
// needs two planes on a one-plane ELF.
if (setup_.pb16i) {
const Q4nxDesign r = designs16[i].role;
c16->pb16i_planes = (r == Q4NX_DESIGN_D || r == Q4NX_DESIGN_O || r == Q4NX_DESIGN_QKVFA_Q) ? 2 : 1;
}
if (!c16->init_elf(*dev_, elf.c_str(), designs16[i].M, designs16[i].K, designs16[i].N, 1, kname.c_str())) {
if (setup_.pb16i) {
err = "pb16i: init_elf failed for " + elf;
return false;
}
std::fprintf(stderr, " [forward] M=16 twin failed to load (%s); "
"this role keeps M=128\n", designs16[i].elf_name);
drop_ctx16(i);
++skipped_n;
continue;
}
++made_n;
std::fprintf(stderr, " [forward] %-7s M=%d K=%d N=%d %s (decode)\n",
q4nx_design_log_tag(designs16[i].role), designs16[i].M, designs16[i].K, designs16[i].N, elf.c_str());
}
note_m16_made(made_n > 0);
if (made) *made = made_n;
if (skipped) *skipped = skipped_n;
return true;
}

bool XdnaTarget::open_head(const Q4nxDesignGeom* designs, int n, std::string& err) {
// run_lm_head packs int8 B tiles from the f32 head and launches them itself, so it
// needs an int8 context with K=H. Under pb16i every decode context is a pb16i one,
Expand Down
12 changes: 12 additions & 0 deletions npu/target/xdna/xdna_target.h
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,18 @@ class XdnaTarget final : public NpuTarget {
// forward and arrives through Setup, because deciding which sidecar file the head has is the container's
// rule and not a target's.
bool open_head(const Q4nxDesignGeom* designs, int n, std::string& err);
// The M=16 decode family (engine#803 A2b): for each role, resolve its `_m16` ELF under Setup::decode_dirs (or
// the sealed pb16i image in the kernels directory, whose instance name comes from q4nx_pb16i_instance), create
// the context and configure it. `made`/`skipped` are what the forward logs its summary line from. A role
// whose ELF is absent keeps its M=128 kernel (a slower path, not a missing one) EXCEPT under pb16i, where the
// int8 weights are not packed for these geometries and the load stops by name. This is the per-role loop that
// lived in Q4nxNpuForward: it is ELF resolution and context creation, which a target owns. The container's
// naming (`sealed_elf`, and the designs' elf_name) still arrives from the forward, and so do the wide and
// narrow sealed families, which the forward loads on top of the contexts this opens.
// `designs16` is mutable: a role with no decode kernel has its M zeroed, which is how the launch path knows
// to stay on M=128 -- the flag lives in the design, not in a parallel array.
bool open_decode_family(Q4nxDesignGeom* designs16, int n, const std::string& sealed_elf, int* made,
int* skipped, std::string& err);
// Whether open_head() found and opened the pb16i ternary head. The model path reads this flag in four
// places, so it stays the forward's; the target writes it.
bool head_is_pb16i() const { return head_pb16i_; }
Expand Down
15 changes: 15 additions & 0 deletions tests/npu_plan_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,21 @@ void test_valid() {
"the pb16i family is read with its sealed ELF");
check(pp.decode_dirs.size() == 2 && pp.decode_dirs[0].under_model, "a model-relative decode dir is kept");

// The search list a target is configured with: the plan's entries, each under the directory it names, in the
// plan's order (engine#1984 review: nothing resolved them, so a plan naming any place but the built-in three
// lost its decode twins without an error).
{
const std::vector<std::string> got = pp.resolved_decode_dirs("/m", "/k");
check(got.size() == 2 && got[0] == "/m/kernels-m16" && got[1] == "/k/.",
"decode dirs resolve under the model dir or the kernel dir, in the plan's order");
auto seq = base_plan();
seq["decode"]["dirs"] = nlohmann::json::array({nlohmann::json{{"root", "model"}, {"path", "kernels-m16-seq"}}});
const std::vector<std::string> only = Plan::load(write_plan("seq", seq)).resolved_decode_dirs("/m", "/k");
check(only.size() == 1 && only[0] == "/m/kernels-m16-seq",
"a plan naming one directory outside the built-in places resolves to exactly that directory");
check(Plan().resolved_decode_dirs("/m", "/k").empty(), "a plan that is not loaded resolves to no directories");
}

// attention.mode is optional and defaults to causal, so a plan written before the field loads and
// still means what it meant then.
check(p.attention_mode == "causal" && !p.noncausal(), "a plan with no attention block is causal");
Expand Down
Loading