Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions npu/forward/include/q4nx_pack.h
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,15 @@ int q4nx_derive_decode_designs(const Q4nxDesignGeom* designs, int count,
* is a key (engine#803 A2), so it uses these; the log keeps the short tag. */
const char* q4nx_design_role_name(Q4nxDesign role);

/* The LOG tag for a design role -- deliberately NOT the unique name above, and deliberately the
* historical mapping (QKVLIN prints as "QKV", every role the old table did not name prints as "D").
*
* It exists so a diagnostic a reader has already learned does not change when code moves: the setup
* side moved into the target (engine#803 A2b) and the log lines had to stay byte-identical, which the
* #1380 review caught them not doing. The mapping is arguably misleading for the MLA and MoE roles --
* that is a change worth making on purpose, in a PR that says so, not as a side effect of a move. */
const char* q4nx_design_log_tag(Q4nxDesign role);

/* Look up the entry for a role in a derived set; NULL if absent. */
const Q4nxDesignGeom* q4nx_find_design(const Q4nxDesignGeom* designs, int count,
Q4nxDesign role);
Expand Down
86 changes: 25 additions & 61 deletions npu/forward/src/q4nx_forward.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -391,26 +391,9 @@ int Q4nxNpuForward::idx_of(Q4nxDesign role) const {
return -1;
}

static const char* role_tag(Q4nxDesign r) {
switch (r) {
case Q4NX_DESIGN_QKV: return "QKV";
case Q4NX_DESIGN_QKV2: return "QKV2";
case Q4NX_DESIGN_O: return "O";
case Q4NX_DESIGN_O2: return "O2";
case Q4NX_DESIGN_GU: return "GU";
case Q4NX_DESIGN_G: return "G";
case Q4NX_DESIGN_U: return "U";
case Q4NX_DESIGN_PLEGATE: return "PLEGATE";
case Q4NX_DESIGN_PLEPROJ: return "PLEPROJ";
case Q4NX_DESIGN_GU2: return "GU2";
case Q4NX_DESIGN_D2: return "D2";
case Q4NX_DESIGN_QKVLIN: return "QKV";
case Q4NX_DESIGN_Z: return "Z";
case Q4NX_DESIGN_QKVFA_Q: return "QKVFAQ";
case Q4NX_DESIGN_QKVFA_KV: return "QKVFAKV";
default: return "D";
}
}
// The log tag lives with the geometry it describes (q4nx_pack.c) so the target's own setup messages and
// this file's can never drift apart; it is the historical mapping, not the unique plan name.
static const char* role_tag(Q4nxDesign r) { return q4nx_design_log_tag(r); }

// --------------------------------------------------------------------------
// init
Expand All @@ -431,7 +414,7 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path,
// The launch layer. A1: everything below that touches a device talks to this, and the settings the
// container declares (weight layout, pb16i image dir, Hadamard fold) are pushed into it as they are
// parsed. A2 turns this into NpuTarget::open(plan, kernels_dir).
xd_ = std::make_unique<npu::target::xdna::XdnaTarget>();
xd_ = std::make_unique<npu::target::xdna::XdnaTarget>(dev);
int had_block = 1024;
bool weight_cache = true;
if (getenv("NPU_NO_WCACHE")) weight_cache = false; // A/B: disable the packed-weight cache
Expand Down Expand Up @@ -1116,46 +1099,27 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path,
}
}

for (int i = 0; i < ndesigns_; i++) {
const Q4nxDesignGeom* g = &designs_[i];
// The two full-attention-QKV-split designs exist only as pb16i kernels (the
// role has no int8 launch that fits), so they get no M=128 context and no
// int8 ELF on disk: their decode context is built below from the _pb16i ELF.
xd_->adopt_geometry(i, *g); // the target owns the geometry it launches on
if (xd_->role_is_pb16i_split(i)) continue; // no int8 launch for these two roles

if (host_gemm_) {
// Offline mode: host GEMMs are plain float matmuls over the decoded weights, so
// no ELF is opened and none is required. A model whose kernel set has not been
// built yet can be loaded and prompted, which is what makes it a usable
// numeric reference for a new geometry (engine #796: the d3-lite backbone
// answers on the host while its six designs do not exist). The artifact is
// named when it is there; the ELF set itself is checked by
// tests/verify_design_elfs + tests/verify_full_elfs.sh, not by this path.
std::string elf = dir + "/" + g->elf_name;
if (file_exists(elf)) {
fprintf(stderr, " [forward] %-3s M=%d K=%d N=%d %s (host-GEMM)\n",
role_tag(g->role), g->M, g->K, g->N, elf.c_str());
} else {
fprintf(stderr, " [forward] %-3s M=%d K=%d N=%d (host-GEMM, no ELF: %s)\n",
role_tag(g->role), g->M, g->K, g->N, g->elf_name);
}
} else {
std::string elf = dir + "/" + g->elf_name;
if (!file_exists(elf)) {
err_ = missing_kernel(dir, g->elf_name, g->role);
return false;
}
I8Ctx* ctx = xd_->make_ctx(i); // adopted: the target owns the context from here
if (!ctx) { err_ = "cannot create a context for role " + std::to_string(i); return false; }
ctx->MD = g->M; ctx->KD = g->K; ctx->ND = g->N;
ctx->blocked_b = weight_layout_b8_;
if (!ctx->init_elf(*dev, elf.c_str(), g->M, g->K, g->N, 1)) {
err_ = "I8Ctx init_elf failed for " + elf;
return false;
}
fprintf(stderr, " [forward] %-3s M=%d K=%d N=%d %s\n", role_tag(g->role),
g->M, g->K, g->N, elf.c_str());
// The M=128 ELF set is the target's to open (engine#803 A2b): it adopts the geometry, probes the
// artifact and refuses by name, and the forward no longer touches an I8Ctx to do it.
{
// What the setup side reads, set once: the remaining two blocks (decode family, lm-head) read
// the same fields, so moving them later needs no new plumbing (engine#803 A2b).
npu::target::xdna::XdnaTarget::Setup setup;
setup.model_dir = mdir;
setup.kernels_dir = dir;
setup.family = family_tag();
setup.plan_path = plan_.loaded() ? plan_path_ : std::string();
setup.b8 = weight_layout_b8_;
setup.pb16i = pb16i_;
setup.t2s = t2s_;
setup.t2s_v2 = t2s_v2_;
setup.host_gemm = host_gemm_;
setup.plan = plan_.loaded() ? &plan_ : nullptr;
xd_->configure(setup);
std::string why;
if (!xd_->open_prefill(designs_, ndesigns_, why)) {
err_ = std::move(why);
return false;
}
}

Expand Down
21 changes: 21 additions & 0 deletions npu/forward/src/q4nx_pack.c
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,27 @@ void q4nx_host_gemm_f32_padded(const float* A, int a_k, int M, int K, int N, con
free(pad);
}

const char* q4nx_design_log_tag(Q4nxDesign role) {
switch (role) {
case Q4NX_DESIGN_QKV: return "QKV";
case Q4NX_DESIGN_QKV2: return "QKV2";
case Q4NX_DESIGN_O: return "O";
case Q4NX_DESIGN_O2: return "O2";
case Q4NX_DESIGN_GU: return "GU";
case Q4NX_DESIGN_G: return "G";
case Q4NX_DESIGN_U: return "U";
case Q4NX_DESIGN_PLEGATE: return "PLEGATE";
case Q4NX_DESIGN_PLEPROJ: return "PLEPROJ";
case Q4NX_DESIGN_GU2: return "GU2";
case Q4NX_DESIGN_D2: return "D2";
case Q4NX_DESIGN_QKVLIN: return "QKV";
case Q4NX_DESIGN_Z: return "Z";
case Q4NX_DESIGN_QKVFA_Q: return "QKVFAQ";
case Q4NX_DESIGN_QKVFA_KV: return "QKVFAKV";
default: return "D";
}
}

const char* q4nx_design_role_name(Q4nxDesign role) {
switch (role) {
case Q4NX_DESIGN_QKV: return "QKV";
Expand Down
78 changes: 77 additions & 1 deletion npu/target/xdna/xdna_target.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -77,9 +77,85 @@ static const char* role_tag(Q4nxDesign r) {

} // namespace

XdnaTarget::XdnaTarget() = default;
// The forward's own probe, kept identical (exists and non-empty): an ELF that is present but zero bytes
// is the failure this catches, and a target that decided differently from its caller would be a second
// definition of "the kernel is there".
static bool elf_present(const std::string& p) {
FILE* f = std::fopen(p.c_str(), "rb");
if (!f) return false;
std::fseek(f, 0, SEEK_END);
const long sz = std::ftell(f);
std::fclose(f);
return sz >= 1;
}

XdnaTarget::XdnaTarget(xrt::device* dev) : dev_(dev) {}
XdnaTarget::~XdnaTarget() = default;

// The M=128 ELF set, moved out of Q4nxNpuForward::init_impl() unchanged (engine#803 A2b): the same
// probe, the same two refusals, the same per-role log line, and the same adopt-then-open order. What
// changed is who owns it -- a missing or unloadable artifact is now refused by the target that would
// launch it, instead of by a caller that then hands it a context.
bool XdnaTarget::open_prefill(const Q4nxDesignGeom* designs, int n, std::string& err) {
const std::string& dir = setup_.kernels_dir;
const bool b8 = setup_.b8;
const bool host_gemm = setup_.host_gemm;
const std::string& plan_path = setup_.plan_path;
if (!designs || n <= 0 || n > Q4NX_MAX_DESIGNS) {
err = "open_prefill: no design set (" + std::to_string(n) + " designs)";
return false;
}
for (int i = 0; i < n; i++) {
const Q4nxDesignGeom* g = &designs[i];
// The two full-attention-QKV-split designs exist only as pb16i kernels (the role has no int8
// launch that fits), so they get no M=128 context and no int8 ELF on disk: their decode context
// is built from the _pb16i ELF by the caller's twin pass.
adopt_geometry(i, *g);
if (role_is_pb16i_split(i)) continue;
const std::string elf = dir + "/" + g->elf_name;
if (host_gemm) {
// Offline mode: host GEMMs are plain float matmuls over the decoded weights, so no ELF is
// opened and none is required. A model whose kernel set is not built yet can still be
// loaded and prompted, which is what makes it a numeric reference for a new geometry
// (engine#796).
if (elf_present(elf))
std::fprintf(stderr, " [forward] %-3s M=%d K=%d N=%d %s (host-GEMM)\n",
q4nx_design_log_tag(g->role), g->M, g->K, g->N, elf.c_str());
else
std::fprintf(stderr, " [forward] %-3s M=%d K=%d N=%d (host-GEMM, no ELF: %s)\n",
q4nx_design_log_tag(g->role), g->M, g->K, g->N, g->elf_name);
continue;
}
if (!elf_present(elf)) {
// The same wording the forward used for this case, because a diagnostic that changes shape
// when code moves is a diagnostic someone has to re-learn.
const char* role = q4nx_design_log_tag(g->role);
err = plan_path.empty() ? ("missing full ELF " + elf)
: ("plan " + plan_path + ": this model's plan needs the " + role +
" kernel \"" + g->elf_name + "\" in " + dir + " (missing)");
return false;
}
if (!dev_) {
err = "open_prefill: no device for the ELF set";
return false;
}
I8Ctx* ctx = make_ctx(i);
if (!ctx) {
err = "cannot create a context for role " + std::to_string(i);
return false;
}
ctx->MD = g->M; ctx->KD = g->K; ctx->ND = g->N;
ctx->blocked_b = b8;
if (!ctx->init_elf(*dev_, elf.c_str(), g->M, g->K, g->N, 1)) {
err = "I8Ctx init_elf failed for " + elf;
return false;
}
std::fprintf(stderr, " [forward] %-3s M=%d K=%d N=%d %s\n", q4nx_design_log_tag(g->role),
g->M, g->K, g->N, elf.c_str());
}
return true;
}

// --------------------------------------------------------------------------
// NpuTarget
// --------------------------------------------------------------------------
Expand Down
42 changes: 41 additions & 1 deletion npu/target/xdna/xdna_target.h
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,10 @@ struct ProfAcc {

class XdnaTarget final : public NpuTarget {
public:
XdnaTarget();
// The device is the XDNA target's business, not the interface's: passing it here is what keeps
// NpuTarget::open(plan, kernels_dir, family, err) exactly as documented for a second target
// (engine#803 A2b). Null is legal and means "host GEMMs only" (init_host()).
explicit XdnaTarget(xrt::device* dev = nullptr);
~XdnaTarget() override;
XdnaTarget(const XdnaTarget&) = delete;
XdnaTarget& operator=(const XdnaTarget&) = delete;
Expand Down Expand Up @@ -110,6 +113,42 @@ class XdnaTarget final : public NpuTarget {
int max_rows() const; // widest launch this target will accept (decode twins included)
int rows_for(int role, int want) const; // rows the phase-selected kernel for `role` can take

// Everything the setup side reads, set once before open_*(). It exists because the remaining two
// blocks (the M=16 decode family and the lm-head context) read a dozen of the forward's fields --
// the plan, its path, the two given directories, the layout family, the sealed ELF, the t2s flag --
// and a twelve-argument call would be a signature nobody can review. Once these live here, moving
// those blocks changes no call site (engine#803 A2b).
struct Setup {
std::string model_dir; // the -m directory
std::string kernels_dir; // the --kernels directory
std::string family; // "", "_b8", "_pb16i", "_t2s", "_t2s2"
std::string plan_path; // the plan this set came from, when one did ("" = derived)
bool b8 = false; // blocked 8x8 tile layout
bool pb16i = false; // integer-exact ternary, sidecar images
bool t2s = false; // 2-bit ternary stream
bool t2s_v2 = false;
bool host_gemm = false; // offline: open no ELF at all
const onebit::npu::plan::Plan* plan = nullptr; // copied into plan_ by configure(); null = none
};
// Setup carries a pointer to the forward's Plan; this copies it, so from here on the target's own
// plan_ is THE copy the setup side reads (open() no longer has to be the one that filled it). Two
// live copies of the plan is the drift the plan layer exists to prevent, so there is one.
void configure(const Setup& s) {
setup_ = s;
if (s.plan) plan_ = *s.plan;
}
const Setup& setup() const { return setup_; }

// The M=128 ELF set: for each design, adopt its geometry, probe its artifact and open it, refusing by
// name when an artifact is missing or will not load (engine#803 A2b). This is the block that used to
// live at the top of Q4nxNpuForward::init_impl(); it is here because it is ELF loading and its
// refusals, which is what a target owns. `host_gemm` opens nothing (the offline path needs no ELF)
// and only names the artifacts it would have used. `b8` is the container's blocked-tile layout.
// `plan_path` is the plan the set came from, when one did: the missing-artifact refusal quotes it, so
// the message a user sees is the same one the forward produced before this moved (an empty string
// means no plan, i.e. a purely derived set).
bool open_prefill(const Q4nxDesignGeom* designs, int n, std::string& err);

// Context construction: the caller configures and initialises the I8Ctx, then it stays here.
I8Ctx* make_ctx(int role);
I8Ctx* ctx(int role) const;
Expand Down Expand Up @@ -150,6 +189,7 @@ class XdnaTarget final : public NpuTarget {
std::string pb16i_stem(int layer, const char* tag) const;

xrt::device* dev_ = nullptr;
Setup setup_; // see configure(): what the setup side reads, in one place
int n_layers_ = 0;
std::string kernels_dir_;
onebit::npu::plan::Plan plan_; // when loaded: what open() checked this target against
Expand Down
Loading