diff --git a/docs/npu.md b/docs/npu.md index 92b38eef2..85857e81f 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -573,6 +573,32 @@ Two 16-row blocks against one 32-row block is 30-31 rows/s against 35-37. The 8- identical with the family off and on, and `NPU_BATCH_CHECK=1` on the wide path reports MATCH on every chunk it checks (32-, 4- and 1-row) in both tickets. Only 32 rows has been measured. +**...and a narrow family for decode (#2073).** A decode step on `bonsai2-27b-t2s2` is 416 launches, and +device wait is 211 of its 302 ms: every launch runs a kernel built for 16 rows and carries one, and a +launch's device time does not depend on how many of its rows are filled. So a pack may also name a sealed +ELF built for FEWER rows: `q4nx_t2s_sealed_narrow_elf` (or `q4nx_pb16i_sealed_narrow_elf`) and +`q4nx_sealed_narrow_rows` (default 4, allowed 1..15) in `config.json`; `NPU_NARROW_SEALED_ELF=` +overrides the file, `=0` switches it off, `NPU_NARROW_ROWS` overrides the rows. A launch of 1..rows rows +takes it (a decode step, or a verify pass of a few rows); anything wider keeps the 16-row family. The rules +are the wide family's: all roles or none, one hw_context, shared weight images, and a file that is missing +or does not load is logged (`[forward] narrow family NOT adopted (...)`) while decode stays on the 16-row +kernels. `narrow_family_complete` and `narrow_takes` in `batch_width.h` are pinned by +`tests/npu_pack_test.cpp`. No pack ships a narrow ELF yet. + +Measured with a 4-row set, one binary, arms 16-row / 4-row / 16-row / 4-row, three prompts generating 49 +tokens each (`(tokens - 1) / (wall - wall of a 1-token request)`), two `npu-hold` tickets: + +| ticket, pair | decode, 16-row family | decode, 4-row family | device wait per step | +|---|---|---|---| +| first, pair 1 | 3.26-3.33 tok/s | **3.69-4.21 tok/s** | 210.1 -> 158.1 ms | +| first, pair 2 (another lane loading the box) | 1.88-2.59 tok/s | 2.33-3.14 tok/s | 230.7 -> 208.1 ms | +| second, pair 1 | 2.85-3.29 tok/s | **3.90-4.16 tok/s** | 234.7 -> 151.7 ms | +| second, pair 2 | 2.15-3.32 tok/s | **4.05-4.19 tok/s** | 210.9 -> 150.9 ms | + +The step wall follows the wait: 286.0 -> 233.7 ms and 296.2 -> 225.9 ms in the quiet pairs. The 49-token +greedy text of each prompt is identical in all eight arms, and `NPU_BATCH_CHECK=1` with the narrow family +reports MATCH on every chunk it checks (1-, 4- and 6-row; the 6-row one runs on the 16-row family). + **A t2s container wants a sealed set for the same reason a pb16i one does.** The layout is not what makes a phase change expensive: the **hw_context** is. Measured on this box in-loop on 2026-11-02, a t2s decode family shipped as eight per-role `_t2s2_m16` ELFs paid **+2.0-2.3 ms on every launch, in every role** — more diff --git a/npu/forward/include/batch_width.h b/npu/forward/include/batch_width.h index 3365c7ff0..ebc9c1920 100644 --- a/npu/forward/include/batch_width.h +++ b/npu/forward/include/batch_width.h @@ -75,7 +75,31 @@ inline int chunk_rows(int narrow, int wide) { return wide > narrow ? wide : narr // launch past `wide` rows is never routed here (the caller refuses it by name). inline bool wide_takes(int rows, int narrow, int wide) { return wide > narrow && rows > narrow && rows <= wide; } -// Which file is the wide sealed ELF, if any? The environment wins over the pack so a set can be tried, or switched +// --- a narrow family for decode (engine: decode device wait) ----------------------------------------------------- +// +// The mirror of the wide family: the same roles built for FEWER rows than the 16-row family, for the launches +// that carry only a few rows (a decode step carries one). Complete means every role with a 16-row kernel has a +// narrow kernel of exactly `narrow` rows, and `narrow` is below every 16-row width; a role without one would +// silently stay on its 16-row kernel, which is safe but is not what the log would claim, so it is all or none. +inline bool narrow_family_complete(const int* base_widths, const int* narrow_widths, int n, int narrow) { + if (!base_widths || !narrow_widths || n <= 0 || narrow <= 0) return false; + int roles = 0; + for (int i = 0; i < n; ++i) { + if (base_widths[i] <= 0) { + if (narrow_widths[i] > 0) return false; + continue; + } + if (narrow_widths[i] != narrow || narrow >= base_widths[i]) return false; + ++roles; + } + return roles > 0; +} + +// Does a launch of `rows` rows go to the narrow family? Only when it fits: 1..narrow rows. One row more and the +// launch would write past the narrow kernel's activation buffer, so the bound is the kernel's own row count. +inline bool narrow_takes(int rows, int narrow) { return narrow > 0 && rows >= 1 && rows <= narrow; } + +// Which file is the wide sealed ELF, if any? (The narrow family's file is chosen by the same rule.) The environment wins over the pack so a set can be tried, or switched // off, without editing a pack: `env` "0" means no wide family even when the pack names one, any other non-empty // value is the path as given. Otherwise the pack's name is used: as given when absolute, else inside `kernels_dir` // (where the 16-row sealed ELF lives). Empty answer = no wide family. `env` may be null. diff --git a/npu/forward/include/q4nx_forward.h b/npu/forward/include/q4nx_forward.h index aac96f583..aeb771585 100644 --- a/npu/forward/include/q4nx_forward.h +++ b/npu/forward/include/q4nx_forward.h @@ -371,6 +371,8 @@ class Q4nxNpuForward { std::string pb16i_sealed_; std::string wide_sealed_; // q4nx__sealed_wide_elf: a wide prefill family (engine#1990) int wide_sealed_rows_ = 32; // q4nx_sealed_wide_rows + std::string narrow_sealed_; // q4nx__sealed_narrow_elf: a narrow decode family + int narrow_sealed_rows_ = 4; // q4nx_sealed_narrow_rows std::string sealed_dir_; // the kernels dir the 16-row sealed ELF was found in // ---- the NPU target (engine#799 P4) ---- // Everything that talks to a device lives behind this: the ELF contexts (M=128 prefill and the M=16 diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 61480b018..e3bebd1b2 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -604,44 +604,64 @@ bool Q4nxNpuForward::open_decode_family_local(bool want_m16, xrt::device* dev, c // All roles or none -- max_batch_rows() is the chunk width, so one role without a wide kernel would be // launched past its buffers. A named file that is missing or does not load is logged and the pass stays at // 16 rows: a pack must never stop serving over an optional speed-up. - const std::string wide_path = - onebit::npu::wide_elf_path(getenv("NPU_WIDE_SEALED_ELF"), wide_sealed_, sealed_dir_); - if (const char* wide_elf = wide_path.c_str(); - !wide_path.empty() && pb16i_ && !pb16i_sealed_.empty() && made > 0) { - const char* wr = getenv("NPU_WIDE_ROWS"); - const int rows = wr && *wr ? atoi(wr) : wide_sealed_rows_; - int wide = 0; + // The narrow decode family is the mirror image: the same instances built for FEWER rows (default 4), taken + // by launches of 1..rows rows -- a decode step carries one. Config `q4nx__sealed_narrow_elf` and + // `q4nx_sealed_narrow_rows`; NPU_NARROW_SEALED_ELF / NPU_NARROW_ROWS override, "0" = off. + // One loader for both: `wide` picks the row bound, the target's make/adopt/drop trio and the log text. + auto load_family = [&](bool wide, const std::string& path, int rows) { + if (path.empty() || !pb16i_ || pb16i_sealed_.empty() || made <= 0) return; + const char* tag = wide ? "wide" : "narrow"; + int n = 0; std::string why; - if (rows <= Q4NX_DESIGN_M_DECODE || rows > Q4NX_DESIGN_M_PREFILL) why = "NPU_WIDE_ROWS must be 17..128"; - else if (!file_exists(wide_elf)) why = std::string("no such file: ") + wide_elf; + if (wide && (rows <= Q4NX_DESIGN_M_DECODE || rows > Q4NX_DESIGN_M_PREFILL)) why = "NPU_WIDE_ROWS must be 17..128"; + else if (!wide && (rows < 1 || rows >= Q4NX_DESIGN_M_DECODE)) why = "NPU_NARROW_ROWS must be 1..15"; + else if (!file_exists(path)) why = "no such file: " + path; for (int i = 0; why.empty() && i < ndesigns_; i++) { I8Ctx* c16 = xd_->ctx16(i); if (!c16) continue; const char* inst = pb16i_instance(designs16_[i].role); - I8Ctx* cw = inst ? xd_->make_ctx_wide(i) : nullptr; - if (!cw) { why = std::string("no wide context for role ") + role_tag(designs16_[i].role); break; } - cw->MD = rows; - cw->KD = c16->KD; - cw->ND = c16->ND; - cw->blocked_b = c16->blocked_b; - cw->pb16i = c16->pb16i; - cw->t2s = c16->t2s; - cw->pb16i_planes = c16->pb16i_planes; - if (!cw->init_elf(*dev, wide_elf, rows, c16->KD, c16->ND, 1, inst)) { + I8Ctx* c = !inst ? nullptr : (wide ? xd_->make_ctx_wide(i) : xd_->make_ctx_narrow(i)); + if (!c) { why = std::string("no ") + tag + " context for role " + role_tag(designs16_[i].role); break; } + c->MD = rows; + c->KD = c16->KD; + c->ND = c16->ND; + c->blocked_b = c16->blocked_b; + c->pb16i = c16->pb16i; + c->t2s = c16->t2s; + c->pb16i_planes = c16->pb16i_planes; + if (!c->init_elf(*dev, path.c_str(), rows, c16->KD, c16->ND, 1, inst)) { why = std::string("init_elf failed for instance ") + inst; break; } - wide++; - } - if (why.empty() && wide > 0 && !xd_->adopt_wide(rows)) why = "a role with a decode context has no wide one"; - if (why.empty() && wide > 0) { - fprintf(stderr, " [forward] wide family: %d role(s) at M=%d from %s (chunks of 17..%d rows)\n", - wide, rows, wide_elf, rows); + n++; + } + if (why.empty() && n > 0 && !(wide ? xd_->adopt_wide(rows) : xd_->adopt_narrow(rows))) + why = std::string("a role with a decode context has no ") + tag + " one"; + if (why.empty() && n > 0) { + if (wide) + fprintf(stderr, " [forward] wide family: %d role(s) at M=%d from %s (chunks of 17..%d rows)\n", + n, rows, path.c_str(), rows); + else + fprintf(stderr, " [forward] narrow family: %d role(s) at M=%d from %s (launches of 1..%d rows)\n", + n, rows, path.c_str(), rows); } else { - xd_->drop_wide(); - fprintf(stderr, " [forward] wide family NOT adopted (%s); chunks stay at %d rows\n", - why.empty() ? "no role has a decode context" : why.c_str(), Q4NX_DESIGN_M_DECODE); + if (wide) xd_->drop_wide(); else xd_->drop_narrow(); + if (wide) + fprintf(stderr, " [forward] wide family NOT adopted (%s); chunks stay at %d rows\n", + why.empty() ? "no role has a decode context" : why.c_str(), Q4NX_DESIGN_M_DECODE); + else + fprintf(stderr, " [forward] narrow family NOT adopted (%s); decode stays on the %d-row kernels\n", + why.empty() ? "no role has a decode context" : why.c_str(), Q4NX_DESIGN_M_DECODE); } + }; + { + const char* wr = getenv("NPU_WIDE_ROWS"); + load_family(true, onebit::npu::wide_elf_path(getenv("NPU_WIDE_SEALED_ELF"), wide_sealed_, sealed_dir_), + wr && *wr ? atoi(wr) : wide_sealed_rows_); + const char* nr = getenv("NPU_NARROW_ROWS"); + load_family(false, + onebit::npu::wide_elf_path(getenv("NPU_NARROW_SEALED_ELF"), narrow_sealed_, sealed_dir_), + nr && *nr ? atoi(nr) : narrow_sealed_rows_); } return true; } @@ -738,6 +758,10 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, wide_sealed_ = read_config_string( cfg_path, t2s_ ? "\"q4nx_t2s_sealed_wide_elf\"" : "\"q4nx_pb16i_sealed_wide_elf\"", ""); wide_sealed_rows_ = q4nx_config_int(cfg_path.c_str(), "q4nx_sealed_wide_rows", 32); + // ... and an optional third one built for fewer rows, for decode steps. + narrow_sealed_ = read_config_string( + cfg_path, t2s_ ? "\"q4nx_t2s_sealed_narrow_elf\"" : "\"q4nx_pb16i_sealed_narrow_elf\"", ""); + narrow_sealed_rows_ = q4nx_config_int(cfg_path.c_str(), "q4nx_sealed_narrow_rows", 4); // A folded container also declares the fold: `q4nx_hadamard` names the sign table and // `q4nx_hadamard_block` the Hadamard block. Without it the sidecar images would be // read in the stored (rotated) basis against un-rotated activations. diff --git a/npu/target/xdna/xdna_target.cpp b/npu/target/xdna/xdna_target.cpp index 9ca41c244..43e6ef982 100644 --- a/npu/target/xdna/xdna_target.cpp +++ b/npu/target/xdna/xdna_target.cpp @@ -366,6 +366,28 @@ bool XdnaTarget::adopt_wide(int rows) { wide_rows_ = rows; return true; } +I8Ctx* XdnaTarget::make_ctx_narrow(int role) { + if (role < 0 || role >= Q4NX_MAX_DESIGNS) return nullptr; + ctxn_[role] = std::make_unique(); + return ctxn_[role].get(); +} +bool XdnaTarget::adopt_narrow(int rows) { + int base[Q4NX_MAX_DESIGNS] = {0}, narrow[Q4NX_MAX_DESIGNS] = {0}; + for (int i = 0; i < Q4NX_MAX_DESIGNS; i++) { + if (ctx16_[i]) base[i] = ctx16_[i]->MD; + if (ctxn_[i]) narrow[i] = ctxn_[i]->MD; + } + if (!pb16i_ || !onebit::npu::narrow_family_complete(base, narrow, Q4NX_MAX_DESIGNS, rows)) { + drop_narrow(); + return false; + } + narrow_rows_ = rows; + return true; +} +void XdnaTarget::drop_narrow() { + for (auto& c : ctxn_) c.reset(); + narrow_rows_ = 0; +} void XdnaTarget::drop_wide() { for (auto& c : ctxw_) c.reset(); wide_rows_ = 0; @@ -394,7 +416,8 @@ void XdnaTarget::note_head(int M, double dpk, double dql, double dwt, double drb int XdnaTarget::design_index(I8Ctx* c) const { if (!c) return -1; for (int i = 0; i < ndesigns_ && i < Q4NX_MAX_DESIGNS; i++) - if (ctx_[i].get() == c || (ctx16_[i] && ctx16_[i].get() == c) || (ctxw_[i] && ctxw_[i].get() == c)) + if (ctx_[i].get() == c || (ctx16_[i] && ctx16_[i].get() == c) || (ctxw_[i] && ctxw_[i].get() == c) || + (ctxn_[i] && ctxn_[i].get() == c)) return i; return -1; } @@ -769,7 +792,12 @@ void XdnaTarget::npu_gemm(const GemvArgs& ar, std::vector& out, std::stri const float as = amax > 0.0f ? amax / 127.0f : 1.0f; // Phase-selected activation tile: a one-row call is decode, so it runs this // role's M=16 twin (one core row instead of 128 rows / 8 live tiles). - I8Ctx& kctx = *ctx_for(&ctx, 1, ar.decode_twin); + I8Ctx* kctx1 = ctx_for(&ctx, 1, ar.decode_twin); + // A decode step carries one row: with a narrow family it runs the kernel built for a few rows instead of 16. + if (pb16i_ && ar.role >= 0 && ar.role < Q4NX_MAX_DESIGNS && ctxn_[ar.role] && + onebit::npu::narrow_takes(1, narrow_rows_)) + kctx1 = ctxn_[ar.role].get(); + I8Ctx& kctx = *kctx1; const bool prof = prof_; const auto tp = prof ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; @@ -1050,6 +1078,9 @@ void XdnaTarget::npu_gemm_rows(const GemvArgs& ar, std::vector& out, std: if (pb16i_ && ar.role >= 0 && ar.role < Q4NX_MAX_DESIGNS && ctxw_[ar.role] && onebit::npu::wide_takes(M, Q4NX_DESIGN_M_DECODE, wide_rows_)) kctxp = ctxw_[ar.role].get(); + else if (pb16i_ && ar.role >= 0 && ar.role < Q4NX_MAX_DESIGNS && ctxn_[ar.role] && + onebit::npu::narrow_takes(M, narrow_rows_)) + kctxp = ctxn_[ar.role].get(); // a short chunk (a verify pass of a few rows) fits the narrow kernel I8Ctx& kctx = *kctxp; const bool prof = prof_; const auto tp = prof ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; diff --git a/npu/target/xdna/xdna_target.h b/npu/target/xdna/xdna_target.h index 687289d98..261f6c197 100644 --- a/npu/target/xdna/xdna_target.h +++ b/npu/target/xdna/xdna_target.h @@ -189,6 +189,13 @@ class XdnaTarget final : public NpuTarget { void drop_wide(); bool adopt_wide(int rows); int wide_rows() const { return wide_rows_; } + // The NARROW family: the same roles built for fewer than 16 rows per launch, for decode steps and other + // launches of 1..narrow_rows() rows. Same rules as the wide family (all roles or none, one sealed hw_context); + // a launch of more rows keeps the 16-row family. + I8Ctx* make_ctx_narrow(int role); + bool adopt_narrow(int rows); + void drop_narrow(); + int narrow_rows() const { return narrow_rows_; } // The context a launch of `rows` rows on `role` would use: the M=16 twin when one was adopted and // the row count fits, otherwise the M=128 one. The forward needs this where it launches a GEMM // itself (the LM head) instead of going through gemv(). @@ -232,6 +239,8 @@ class XdnaTarget final : public NpuTarget { std::unique_ptr ctx16_[Q4NX_MAX_DESIGNS]; std::unique_ptr ctxw_[Q4NX_MAX_DESIGNS]; // the wide pb16i/t2s family (engine#1990) int wide_rows_ = 0; // 0 = no wide family + std::unique_ptr ctxn_[Q4NX_MAX_DESIGNS]; // the narrow pb16i/t2s family (decode) + int narrow_rows_ = 0; // 0 = no narrow family std::unique_ptr ctx_head_; bool m16_ = false; bool m16_check_ = false; diff --git a/tests/npu_pack_test.cpp b/tests/npu_pack_test.cpp index 879e43ec6..5fdaec024 100644 --- a/tests/npu_pack_test.cpp +++ b/tests/npu_pack_test.cpp @@ -151,6 +151,25 @@ static void wide_family_arms() { check(wide_elf_path("0", "W.elf", "/k").empty(), "wide elf: NPU_WIDE_SEALED_ELF=0 switches a pack's wide family off"); check(wide_elf_path("0x.elf", "", "/k") == "0x.elf", "wide elf: only the exact value 0 means off"); std::printf(" wide family: 21 arms (complete or not, chunk width, which launch takes it, which file)\n"); + // The narrow (decode) family: the mirror rules. + using onebit::npu::narrow_family_complete; + using onebit::npu::narrow_takes; + const int n4[] = {4, 4, 0, 4}; + check(narrow_family_complete(n16, n4, 4, 4), "narrow family: every decode role has a 4-row kernel -> complete"); + const int nhole[] = {4, 0, 0, 4}; + check(!narrow_family_complete(n16, nhole, 4, 4), "narrow family: one role without a narrow kernel -> not complete"); + const int nmixed[] = {4, 8, 0, 4}; + check(!narrow_family_complete(n16, nmixed, 4, 4), "narrow family: a role built for another row count -> not complete"); + const int nextra[] = {4, 4, 4, 4}; + check(!narrow_family_complete(n16, nextra, 4, 4), "narrow family: a kernel for a role with no decode one -> refused"); + check(!narrow_family_complete(n16, n16, 4, 16), "narrow family: 'narrow' must be narrower than the decode family"); + check(!narrow_family_complete(n16, w32, 4, 32), "narrow family: a wide set is not a narrow one"); + check(!narrow_family_complete(nullptr, n4, 4, 4) && !narrow_family_complete(n16, n4, 0, 4) && + !narrow_family_complete(n16, n4, 4, 0), "narrow family: null, empty and zero-row inputs are not complete"); + check(narrow_takes(1, 4) && narrow_takes(4, 4), "narrow takes: 1 and 4 rows go to a 4-row kernel"); + check(!narrow_takes(5, 4) && !narrow_takes(16, 4), "narrow takes: 5 rows never go to a 4-row kernel"); + check(!narrow_takes(1, 0) && !narrow_takes(0, 4), "narrow takes: no family takes nothing, and 0 rows is not a launch"); + std::printf(" narrow family: 10 arms (complete or not, which launch takes it)\n"); } int main(int argc, char** argv) {