Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 5 additions & 96 deletions npu/forward/src/q4nx_forward.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -121,10 +121,6 @@ static const uint8_t* data_of(ModelWeights* mw, const TensorDesc* d, size_t byte
return (const uint8_t*)model_tensor_data(mw, const_cast<TensorDesc*>(d));
}

// Which instance of the sealed ELF runs a role, or nullptr when the sealed ELF does not cover it. The table
// moved to q4nx_pack.c (q4nx_pb16i_instance) because the target opens the decode family (engine#803 A2b) and a
// contract both sides must agree on cannot live on one side; this wrapper keeps the call sites unchanged.
static const char* pb16i_instance(Q4nxDesign r) { return q4nx_pb16i_instance(r); }

// Where an M=16 int8 twin is looked for, first hit wins: <model dir>/kernels-m16,
// <kernels>/m16, then <kernels> itself.
Expand Down Expand Up @@ -1311,99 +1307,12 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path,
snprintf(designs16_[i].elf_name, Q4NX_ELF_NAME_MAX, "%s", n16.c_str());
}
}
std::vector<std::string> m16dirs;
plan_decode_dirs_.clear();
if (plan_.loaded() && !plan_.decode_dirs.empty()) {
// The plan says which of the two given paths each candidate hangs off, so the runtime
// stops guessing which base a bare directory name meant.
for (const auto& d : plan_.decode_dirs)
plan_decode_dirs_.push_back((d.under_model ? mdir : dir) + "/" + d.path);
for (const auto& p : plan_decode_dirs_) m16dirs.push_back(p);
} else {
m16dirs.push_back(mdir + "/kernels-m16");
m16dirs.push_back(dir + "/m16");
m16dirs.push_back(dir);
}
// The decode family is the target's (engine#803 A2b): it resolves each role's `_m16` ELF under the plan's
// decode directories (or the sealed pb16i image), creates the context and configures it. What stays here is
// the container's naming -- which sidecar file the sealed ELF is -- and the summary line, which is about the
// model's geometry rather than the target's.
int made = 0, skipped = 0;
for (int i = 0; i < ndesigns_; i++) {
if (designs16_[i].M <= 0) { skipped++; continue; }
std::string elf, kname = "main:seq";
if (pb16i_ && !pb16i_sealed_.empty()) {
// One ELF for every ternary phase: the file is the same for all roles and
// only the instance name differs, so I8Ctx reuses the one hw_context.
const char* inst = pb16i_instance(designs16_[i].role);
if (!inst) {
fprintf(stderr, " [forward] pb16i: the sealed ELF has no instance for "
"role %s; this role has no decode kernel\n",
role_tag(designs16_[i].role));
designs16_[i].M = 0;
skipped++;
continue;
}
elf = dir + "/" + pb16i_sealed_;
kname = inst;
if (!file_exists(elf)) {
err_ = plan_.loaded() ? ("plan " + plan_path_ + ": decode.sealed_elf \"" +
pb16i_sealed_ + "\" is not in " + dir + " (missing)")
: ("pb16i: missing sealed ELF " + elf);
return false;
}
} else {
for (const std::string& d : m16dirs) {
std::string cand = d + "/" + designs16_[i].elf_name;
if (file_exists(cand)) { elf = cand; break; }
}
}
if (elf.empty()) {
// A pb16i role without its image kernel must stop the load: it is not
// a slower path, it is a missing one (the int8 weights are not packed
// for these geometries under this layout).
if (pb16i_) {
err_ = plan_.loaded()
? std::string("plan ") + plan_path_ + ": decode role " +
role_tag(designs16_[i].role) + " needs \"" + designs16_[i].elf_name +
"\" (missing)"
: std::string("pb16i: missing decode ELF ") + designs16_[i].elf_name;
return false;
}
skipped++;
continue; // stay on M=128
}
I8Ctx* c16 = xd_->make_ctx16(i, designs16_[i]);
if (!c16) { err_ = "cannot create the decode context for role " + std::to_string(i); return false; }
c16->MD = designs16_[i].M;
c16->KD = designs16_[i].K;
c16->ND = designs16_[i].N;
c16->blocked_b = weight_layout_b8_;
c16->pb16i = pb16i_;
c16->t2s = t2s_;
// Which pb16i ELFs leave two int32 planes (the arena's ACC64 family: D, O and
// the full-attention q half). This mirrors the ELF family contract rather
// than inferring it, and load/launch refuse a tensor that needs two planes on
// a one-plane ELF.
if (pb16i_) {
const Q4nxDesign r = designs16_[i].role;
c16->pb16i_planes =
(r == Q4NX_DESIGN_D || r == Q4NX_DESIGN_O || r == Q4NX_DESIGN_QKVFA_Q) ? 2 : 1;
}
if (!c16->init_elf(*dev, elf.c_str(), designs16_[i].M,
designs16_[i].K, designs16_[i].N, 1, kname.c_str())) {
if (pb16i_) {
err_ = "pb16i: init_elf failed for " + elf;
return false;
}
fprintf(stderr, " [forward] M=16 twin failed to load (%s); "
"this role keeps M=128\n", designs16_[i].elf_name);
xd_->drop_ctx16(i);
skipped++;
continue;
}
made++;
fprintf(stderr, " [forward] %-7s M=%d K=%d N=%d %s (decode)\n",
role_tag(designs_[i].role), designs16_[i].M, designs16_[i].K,
designs16_[i].N, elf.c_str());
}
xd_->note_m16_made(made > 0);
if (!xd_->open_decode_family(designs16_, ndesigns_, pb16i_sealed_, &made, &skipped, err_)) return false;
if (pb16i_ && made < ndesigns_ - 0) {
fprintf(stderr, " [forward] %s: %d of %d designs have a decode kernel\n",
t2s_ ? "t2s" : "pb16i", made, ndesigns_);
Expand Down
96 changes: 96 additions & 0 deletions npu/target/xdna/xdna_target.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -541,6 +541,102 @@ void XdnaTarget::prof_dump() {
prof_head_ = ProfAcc{};
}

namespace {
bool decode_elf_exists(const std::string& p) {
if (p.empty()) return false;
std::error_code ec;
const auto sz = std::filesystem::file_size(p, ec);
return !ec && sz > 0;
}
} // namespace

bool XdnaTarget::open_decode_family(Q4nxDesignGeom* designs16, int n, const std::string& sealed_elf,
int* made, int* skipped, std::string& err) {
int made_n = 0, skipped_n = 0;
for (int i = 0; i < n; i++) {
if (designs16[i].M <= 0) { ++skipped_n; continue; }
std::string elf, kname = "main:seq";
if (setup_.pb16i && !sealed_elf.empty()) {
// One ELF for every ternary phase: the file is the same for all roles and only the instance name
// differs, so I8Ctx reuses the one hw_context.
const char* inst = q4nx_pb16i_instance(designs16[i].role);
if (!inst) {
std::fprintf(stderr,
" [forward] pb16i: the sealed ELF has no instance for role %s; this role has no "
"decode kernel\n",
q4nx_design_log_tag(designs16[i].role));
designs16[i].M = 0;
++skipped_n;
continue;
}
elf = setup_.kernels_dir + "/" + sealed_elf;
kname = inst;
if (!decode_elf_exists(elf)) {
err = plan_.loaded() ? ("plan " + setup_.plan_path + ": decode.sealed_elf \"" + sealed_elf +
"\" is not in " + setup_.kernels_dir + " (missing)")
: ("pb16i: missing sealed ELF " + elf);
return false;
}
} else {
for (const std::string& d : setup_.decode_dirs) {
const std::string cand = d + "/" + designs16[i].elf_name;
if (decode_elf_exists(cand)) { elf = cand; break; }
}
}
if (elf.empty()) {
// A pb16i role without its image kernel must stop the load: it is not a slower path, it is a missing
// one (the int8 weights are not packed for these geometries under this layout).
if (setup_.pb16i) {
err = plan_.loaded() ? std::string("plan ") + setup_.plan_path + ": decode role " +
q4nx_design_log_tag(designs16[i].role) + " needs \"" +
designs16[i].elf_name + "\" (missing)"
: std::string("pb16i: missing decode ELF ") + designs16[i].elf_name;
return false;
}
++skipped_n;
continue; // stay on M=128
}
I8Ctx* c16 = make_ctx16(i, designs16[i]);
if (!c16) {
err = "cannot create the decode context for role " + std::to_string(i);
return false;
}
c16->MD = designs16[i].M;
c16->KD = designs16[i].K;
c16->ND = designs16[i].N;
c16->blocked_b = setup_.b8;
c16->pb16i = setup_.pb16i;
c16->t2s = setup_.t2s;
// Which pb16i ELFs leave two int32 planes (the arena's ACC64 family: D, O and the full-attention q half).
// This mirrors the ELF family contract rather than inferring it, and load/launch refuse a tensor that
// needs two planes on a one-plane ELF.
if (setup_.pb16i) {
const Q4nxDesign r = designs16[i].role;
c16->pb16i_planes = (r == Q4NX_DESIGN_D || r == Q4NX_DESIGN_O || r == Q4NX_DESIGN_QKVFA_Q) ? 2 : 1;
}
if (!c16->init_elf(*dev_, elf.c_str(), designs16[i].M, designs16[i].K, designs16[i].N, 1, kname.c_str())) {
if (setup_.pb16i) {
err = "pb16i: init_elf failed for " + elf;
return false;
}
std::fprintf(stderr,
" [forward] M=16 twin failed to load (%s); this role keeps M=128\n",
designs16[i].elf_name);
drop_ctx16(i);
++skipped_n;
continue;
}
++made_n;
std::fprintf(stderr, " [forward] %-7s M=%d K=%d N=%d %s (decode)\n",
q4nx_design_log_tag(designs16[i].role), designs16[i].M, designs16[i].K, designs16[i].N,
elf.c_str());
}
note_m16_made(made_n > 0);
if (made) *made = made_n;
if (skipped) *skipped = skipped_n;
return true;
}

bool XdnaTarget::open_head(const Q4nxDesignGeom* designs, int n, std::string& err) {
// run_lm_head packs int8 B tiles from the f32 head and launches them itself, so it
// needs an int8 context with K=H. Under pb16i every decode context is a pb16i one,
Expand Down
11 changes: 11 additions & 0 deletions npu/target/xdna/xdna_target.h
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,17 @@ class XdnaTarget final : public NpuTarget {
// forward and arrives through Setup, because deciding which sidecar file the head has is the container's
// rule and not a target's.
bool open_head(const Q4nxDesignGeom* designs, int n, std::string& err);
// The M=16 decode family (engine#803 A2b): for each role, resolve its `_m16` ELF under the plan's decode
// directories (or the sealed pb16i image, whose instance name comes from q4nx_pb16i_instance), create the
// context and configure it. `made`/`skipped` are what the forward still logs the summary from; a role whose
// ELF is absent keeps its M=128 kernel (a slower path, not a missing one) EXCEPT under pb16i, where the int8
// weights are not packed for these geometries and the load must stop by name. This is the block that used to
// live in Q4nxNpuForward::init_impl -- here because it is ELF resolution and context creation, which is what a
// target owns; the container's naming (`sealed_elf`) still arrives from the forward.
// `designs16` is mutable: a role with no decode kernel has its M zeroed ("no decode design"), which is how the
// launch path knows to stay on M=128 -- the flag lives in the design, not in a parallel array.
bool open_decode_family(Q4nxDesignGeom* designs16, int n, const std::string& sealed_elf, int* made,
int* skipped, std::string& err);
// Whether open_head() found and opened the pb16i ternary head. The model path reads this flag in four
// places, so it stays the forward's; the target writes it.
bool head_is_pb16i() const { return head_pb16i_; }
Expand Down
Loading