From 6080e4724071b63ae0ac5e43cd64508eecbaffd9 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 18:01:19 -0400 Subject: [PATCH 1/2] npu: a sealed set can carry a wide prefill family (NPU_WIDE_SEALED_ELF, off by default): 28.3-30.8 -> 35.0-35.9 tok/s to first token with a 32-row set (Refs #1990) A sealed pb16i/t2s set is built for 16 rows per launch, so a prefill chunk is 16 rows. NPU_WIDE_SEALED_ELF names a second sealed ELF with the same instances built for NPU_WIDE_ROWS rows (default 32). When it loads for every role that has a 16-row kernel, the chunk width becomes that row count and launches of more than 16 rows take the wide kernel; decode and chunks of up to 16 rows stay on the 16-row family. All roles or none, and a launch carrying more rows than its kernel holds is refused by name. The three rules (wide_family_complete, chunk_rows, wide_takes) live in batch_width.h and are pinned without a device by 14 new arms in tests/npu_pack_test.cpp. Experimental: no pack ships a wide ELF and the plan has no key for one, so it is an environment switch. Co-Authored-By: Claude Fable 5.2 --- docs/npu.md | 18 +++++++++++ npu/forward/include/batch_width.h | 31 +++++++++++++++++++ npu/forward/src/q4nx_forward.cpp | 42 ++++++++++++++++++++++++++ npu/target/xdna/xdna_target.cpp | 50 +++++++++++++++++++++++++++++-- npu/target/xdna/xdna_target.h | 13 ++++++++ tests/npu_pack_test.cpp | 33 ++++++++++++++++++++ 6 files changed, 184 insertions(+), 3 deletions(-) diff --git a/docs/npu.md b/docs/npu.md index 2448c851c..8332eae33 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -538,6 +538,24 @@ second pair is not quoted as a speedup. `NPU_DEQ_LDEXP=2` runs both forms and co `tests/npu_pb16i_layout_test.cpp` holds the two forms to the same bits for one and two planes, the edges of the exponent range and exponents outside it. The `[bl]` line names the term as `deq`. +**A sealed set may carry a wide family for prefill (#1990, experimental, off by default).** The served +sealed sets are built for 16 rows per launch, so a prefill chunk is 16 rows and a 536-token prompt is 34 +blocks of 416 launches. `NPU_WIDE_SEALED_ELF=` names a second sealed ELF with the same instances +built for `NPU_WIDE_ROWS` rows (default 32, allowed 17..128). When it loads for every role that has a +16-row kernel, the chunk width becomes that row count and a launch of more than 16 rows takes the wide +kernel; decode steps and chunks of up to 16 rows stay on the 16-row family. It is all roles or none: the +chunk width is one number for the pass, so a set missing one role is not adopted and the serve log says +why (`[forward] wide family NOT adopted (...)`). The weight images are shared, only the activation and +result buffers differ. A launch that carries more rows than its kernel holds is refused by name. The rules +(`wide_family_complete`, `chunk_rows`, `wide_takes` in `npu/forward/include/batch_width.h`) are pinned +without a device by `tests/npu_pack_test.cpp`. No pack ships a wide ELF and the plan has no key for one +yet, which is why this is an environment switch. Measured on `bonsai2-27b-t2s2` with a 32-row set, one +`npu-hold` ticket, one binary, arms default / wide / default / wide: three 510-536-token prompts reached their first token at 28.3-29.8 and 30.5-30.8 tok/s +on the default path and **35.6-35.9 and 35.0-35.4 tok/s** with the wide set. The block median was +534.2 and 509.3 ms for 16 rows (103 blocks per arm) against 871.0 and 881.5 ms for 32 rows (50 blocks per +arm): 30.0-31.4 rows/s -> **36.3-36.7 rows/s**. The 8-token greedy text is identical in every arm, and `NPU_BATCH_CHECK=1` on the wide +path reports MATCH on every chunk it checks (32-, 4- and 1-row). + **A t2s container wants a sealed set for the same reason a pb16i one does.** The layout is not what makes a phase change expensive: the **hw_context** is. Measured on this box in-loop on 2026-11-02, a t2s decode family shipped as eight per-role `_t2s2_m16` ELFs paid **+2.0-2.3 ms on every launch, in every role** — more diff --git a/npu/forward/include/batch_width.h b/npu/forward/include/batch_width.h index 9901849a9..3ad110da9 100644 --- a/npu/forward/include/batch_width.h +++ b/npu/forward/include/batch_width.h @@ -42,5 +42,36 @@ inline int batch_width_min(const int* widths, int n, int fallback) { return w > 0 ? w : (fallback > 0 ? fallback : 1); } +// --- a wide family beside the 16-row one (engine#1990) --------------------------------------------------------- +// +// A sealed set may carry the same roles twice: built for `narrow` rows (decode and short chunks) and for `wide` rows +// (prefill chunks). The three questions below are the ones that can be wrong in the dangerous direction, so they +// live here with the minimum above and are pinned by the same host test. + +// Is every role that has a narrow kernel also covered by a wide kernel of exactly `wide` rows? The chunk width is +// one number for the whole pass, so a single role without a wide kernel means the wide family cannot be used at +// all: its launches would carry more rows than that role's buffers hold. `wide` must exceed every narrow width. +inline bool wide_family_complete(const int* narrow_widths, const int* wide_widths, int n, int wide) { + if (!narrow_widths || !wide_widths || n <= 0 || wide <= 0) return false; + int roles = 0; + for (int i = 0; i < n; ++i) { + if (narrow_widths[i] <= 0) { + if (wide_widths[i] > 0) return false; // a wide kernel for a role the set does not run + continue; + } + if (wide_widths[i] != wide || wide <= narrow_widths[i]) return false; + ++roles; + } + return roles > 0; +} + +// The chunk width of a pass: the wide family's rows when one was adopted (0 = none), else the narrow minimum. +inline int chunk_rows(int narrow, int wide) { return wide > narrow ? wide : narrow; } + +// Does a launch of `rows` rows go to the wide family? Only when it does not fit the narrow one and does fit the +// wide one: a wide launch costs more than a narrow one whatever it carries, so a decode step never takes it, and a +// launch past `wide` rows is never routed here (the caller refuses it by name). +inline bool wide_takes(int rows, int narrow, int wide) { return wide > narrow && rows > narrow && rows <= wide; } + } // namespace npu } // namespace onebit diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 18deb4d7b..4ee9fed39 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -1412,6 +1412,48 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, fprintf(stderr, " [forward] decode family: %d role(s) at M=%d, %d on M=%d%s\n", made, Q4NX_DESIGN_M_DECODE, skipped, Q4NX_DESIGN_M_PREFILL, want_m16 ? "" : " (NPU_DECODE_M16=0: disabled)"); + // engine#1990: a wide prefill family for a sealed pb16i/t2s set. NPU_WIDE_SEALED_ELF names a second sealed + // ELF carrying the same instances built for NPU_WIDE_ROWS rows (default 32); a chunk of more than 16 rows + // then runs one launch per role instead of two. Experimental and off unless the variable is set: no pack + // ships such an ELF yet, and the plan has no key for it. All roles or none -- max_batch_rows() is the chunk + // width, so one role without a wide kernel would be launched past its buffers. + if (const char* wide_elf = getenv("NPU_WIDE_SEALED_ELF"); + wide_elf && *wide_elf && pb16i_ && !pb16i_sealed_.empty() && made > 0) { + const char* wr = getenv("NPU_WIDE_ROWS"); + const int rows = wr && *wr ? atoi(wr) : 32; + int wide = 0; + std::string why; + if (rows <= Q4NX_DESIGN_M_DECODE || rows > Q4NX_DESIGN_M_PREFILL) why = "NPU_WIDE_ROWS must be 17..128"; + else if (!file_exists(wide_elf)) why = std::string("no such file: ") + wide_elf; + for (int i = 0; why.empty() && i < ndesigns_; i++) { + I8Ctx* c16 = xd_->ctx16(i); + if (!c16) continue; + const char* inst = pb16i_instance(designs16_[i].role); + I8Ctx* cw = inst ? xd_->make_ctx_wide(i) : nullptr; + if (!cw) { why = std::string("no wide context for role ") + role_tag(designs16_[i].role); break; } + cw->MD = rows; + cw->KD = c16->KD; + cw->ND = c16->ND; + cw->blocked_b = c16->blocked_b; + cw->pb16i = c16->pb16i; + cw->t2s = c16->t2s; + cw->pb16i_planes = c16->pb16i_planes; + if (!cw->init_elf(*dev, wide_elf, rows, c16->KD, c16->ND, 1, inst)) { + why = std::string("init_elf failed for instance ") + inst; + break; + } + wide++; + } + if (why.empty() && wide > 0 && !xd_->adopt_wide(rows)) why = "a role with a decode context has no wide one"; + if (why.empty() && wide > 0) { + fprintf(stderr, " [forward] wide family: %d role(s) at M=%d from %s (chunks of 17..%d rows)\n", + wide, rows, wide_elf, rows); + } else { + xd_->drop_wide(); + fprintf(stderr, " [forward] wide family NOT adopted (%s); chunks stay at %d rows\n", + why.empty() ? "no role has a decode context" : why.c_str(), Q4NX_DESIGN_M_DECODE); + } + } } // The lm head's own context (pb16i ternary images, the int8 ladder, or the q4_1 gemv head). Cut out diff --git a/npu/target/xdna/xdna_target.cpp b/npu/target/xdna/xdna_target.cpp index 5864d12e2..9ca41c244 100644 --- a/npu/target/xdna/xdna_target.cpp +++ b/npu/target/xdna/xdna_target.cpp @@ -295,7 +295,10 @@ int XdnaTarget::max_rows() const { if (pb16i_) for (int i = 0; i < n; i++) if (ctx16_[i]) { widths16[i] = ctx16_[i]->MD; any16 = true; } - if (any16) return onebit::npu::batch_width_min(widths16, n, 1); + if (any16) { + // engine#1990: with a wide family adopted for every role, a chunk may be as wide as that family. + return onebit::npu::chunk_rows(onebit::npu::batch_width_min(widths16, n, 1), wide_rows_); + } int widths[Q4NX_MAX_DESIGNS] = {0}; bool any_ctx = false; for (int i = 0; i < n; i++) @@ -315,6 +318,8 @@ int XdnaTarget::max_rows() const { int XdnaTarget::rows_for(int role, int want) const { if (role < 0 || role >= Q4NX_MAX_DESIGNS) return want; if (m16_ && want >= 1 && want <= 16 && ctx16_[role]) return ctx16_[role]->MD; + if (pb16i_ && ctxw_[role] && onebit::npu::wide_takes(want, Q4NX_DESIGN_M_DECODE, wide_rows_)) + return ctxw_[role]->MD; // engine#1990 return ctx_[role] ? ctx_[role]->MD : want; } @@ -339,6 +344,32 @@ I8Ctx* XdnaTarget::ctx16(int role) const { if (role < 0 || role >= Q4NX_MAX_DESIGNS) return nullptr; return ctx16_[role].get(); } +I8Ctx* XdnaTarget::make_ctx_wide(int role) { + if (role < 0 || role >= Q4NX_MAX_DESIGNS) return nullptr; + ctxw_[role] = std::make_unique(); + return ctxw_[role].get(); +} +I8Ctx* XdnaTarget::ctx_wide(int role) const { + if (role < 0 || role >= Q4NX_MAX_DESIGNS) return nullptr; + return ctxw_[role].get(); +} +bool XdnaTarget::adopt_wide(int rows) { + int narrow[Q4NX_MAX_DESIGNS] = {0}, wide[Q4NX_MAX_DESIGNS] = {0}; + for (int i = 0; i < Q4NX_MAX_DESIGNS; i++) { + if (ctx16_[i]) narrow[i] = ctx16_[i]->MD; + if (ctxw_[i]) wide[i] = ctxw_[i]->MD; + } + if (!pb16i_ || !onebit::npu::wide_family_complete(narrow, wide, Q4NX_MAX_DESIGNS, rows)) { + drop_wide(); + return false; + } + wide_rows_ = rows; + return true; +} +void XdnaTarget::drop_wide() { + for (auto& c : ctxw_) c.reset(); + wide_rows_ = 0; +} void XdnaTarget::drop_ctx16(int role) { if (role >= 0 && role < Q4NX_MAX_DESIGNS) ctx16_[role].reset(); } @@ -363,7 +394,8 @@ void XdnaTarget::note_head(int M, double dpk, double dql, double dwt, double drb int XdnaTarget::design_index(I8Ctx* c) const { if (!c) return -1; for (int i = 0; i < ndesigns_ && i < Q4NX_MAX_DESIGNS; i++) - if (ctx_[i].get() == c || (ctx16_[i] && ctx16_[i].get() == c)) return i; + if (ctx_[i].get() == c || (ctx16_[i] && ctx16_[i].get() == c) || (ctxw_[i] && ctxw_[i].get() == c)) + return i; return -1; } @@ -1011,11 +1043,23 @@ void XdnaTarget::npu_gemm_rows(const GemvArgs& ar, std::vector& out, std: // Rows > 1 is prefill and keeps the M=128 kernel (one launch per chunk); a // single row here is a decode step and gets the M=16 twin, exactly like // npu_gemm(). - I8Ctx& kctx = *ctx_for(&ctx, M, ar.decode_twin); + I8Ctx* kctxp = ctx_for(&ctx, M, ar.decode_twin); + // engine#1990: more rows than the 16-row family holds go to the wide family when the set has one. The weight + // BO, its e_ref and its mode are the launch's (layer, key) slot, not the context's, so both families read the + // same image; only the row count of the activation and result buffers differs. + if (pb16i_ && ar.role >= 0 && ar.role < Q4NX_MAX_DESIGNS && ctxw_[ar.role] && + onebit::npu::wide_takes(M, Q4NX_DESIGN_M_DECODE, wide_rows_)) + kctxp = ctxw_[ar.role].get(); + I8Ctx& kctx = *kctxp; const bool prof = prof_; const auto tp = prof ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; if (pb16i_ && kctx.pb16i) { out.assign((size_t)M * (size_t)N, 0.0f); // sized before every refusal path below + if (M > kctx.MD) { + // Never launch more rows than the kernel's buffers hold: that reads and writes past them (engine#1372). + err = "pb16i: " + std::to_string(M) + " rows on a " + std::to_string(kctx.MD) + "-row kernel"; + return; + } if (!sc) { err = "pb16i: no sidecar tag for this projection"; return; } PackedWeight* slot = nullptr; if (weight_cache_ && wkey >= 0 && ar.layer >= 0 && ar.layer < n_layers_ && dev_) { diff --git a/npu/target/xdna/xdna_target.h b/npu/target/xdna/xdna_target.h index 3ba8ae2ce..687289d98 100644 --- a/npu/target/xdna/xdna_target.h +++ b/npu/target/xdna/xdna_target.h @@ -178,6 +178,17 @@ class XdnaTarget final : public NpuTarget { std::unique_ptr& head_ctx_ref(); void note_m16_made(bool any) { m16_ = m16_ || any; } void drop_ctx16(int role); + // engine#1990: a pb16i/t2s set's WIDE family -- the same roles built for more than 16 rows per launch, one + // sealed hw_context like the decode family. A launch of more than 16 rows takes it; decode and short chunks + // keep the 16-row family (a wide launch costs more than a 16-row one whatever it carries, so one row must + // never run on it). adopt_wide() takes the family only if EVERY role that has a decode context has a wide + // one of exactly `rows` rows (batch_width.h: wide_family_complete); otherwise it drops every wide context + // and answers false. max_rows() is the chunk width, and a chunk has to be launchable on all of its roles. + I8Ctx* make_ctx_wide(int role); + I8Ctx* ctx_wide(int role) const; + void drop_wide(); + bool adopt_wide(int rows); + int wide_rows() const { return wide_rows_; } // The context a launch of `rows` rows on `role` would use: the M=16 twin when one was adopted and // the row count fits, otherwise the M=128 one. The forward needs this where it launches a GEMM // itself (the LM head) instead of going through gemv(). @@ -219,6 +230,8 @@ class XdnaTarget final : public NpuTarget { std::unique_ptr ctx_[Q4NX_MAX_DESIGNS]; Q4nxDesignGeom designs16_[Q4NX_MAX_DESIGNS]; std::unique_ptr ctx16_[Q4NX_MAX_DESIGNS]; + std::unique_ptr ctxw_[Q4NX_MAX_DESIGNS]; // the wide pb16i/t2s family (engine#1990) + int wide_rows_ = 0; // 0 = no wide family std::unique_ptr ctx_head_; bool m16_ = false; bool m16_check_ = false; diff --git a/tests/npu_pack_test.cpp b/tests/npu_pack_test.cpp index e8f354a95..16d508fda 100644 --- a/tests/npu_pack_test.cpp +++ b/tests/npu_pack_test.cpp @@ -112,8 +112,41 @@ static void batch_width_arms() { std::printf(" batch width: 7 arms (min not first, empty slots, fallback)\n"); } +/* engine#1990: a sealed set may carry a wide family beside the 16-row one. The rules that decide whether it is used, + * how wide a chunk is and which launches take it are pure, and wrong in the dangerous direction means more rows in a + * launch than a kernel's buffers hold. */ +static void wide_family_arms() { + using onebit::npu::chunk_rows; + using onebit::npu::wide_family_complete; + using onebit::npu::wide_takes; + const int n16[] = {16, 16, 0, 16}; + const int w32[] = {32, 32, 0, 32}; + check(wide_family_complete(n16, w32, 4, 32), "wide family: every decode role has a 32-row kernel -> complete"); + const int hole[] = {32, 0, 0, 32}; + check(!wide_family_complete(n16, hole, 4, 32), "wide family: one decode role without a wide kernel -> not complete"); + const int mixed[] = {32, 64, 0, 32}; + check(!wide_family_complete(n16, mixed, 4, 32), "wide family: a role built for another row count -> not complete"); + const int extra[] = {32, 32, 32, 32}; + check(!wide_family_complete(n16, extra, 4, 32), "wide family: a wide kernel for a role with no decode one -> refused"); + const int w16[] = {16, 16, 0, 16}; + check(!wide_family_complete(n16, w16, 4, 16), "wide family: 'wide' must be wider than the decode family"); + const int none[] = {0, 0, 0, 0}; + check(!wide_family_complete(none, none, 4, 32), "wide family: no role at all is not a family"); + check(!wide_family_complete(nullptr, w32, 4, 32) && !wide_family_complete(n16, w32, 0, 32) && + !wide_family_complete(n16, w32, 4, 0), "wide family: null, empty and zero-row inputs are not complete"); + check(chunk_rows(16, 0) == 16, "chunk rows: no wide family keeps the decode width"); + check(chunk_rows(16, 32) == 32, "chunk rows: an adopted wide family widens the chunk"); + check(chunk_rows(16, 8) == 16, "chunk rows: a 'wide' value below the decode width never narrows the chunk"); + check(!wide_takes(1, 16, 32) && !wide_takes(16, 16, 32), "wide takes: 1 and 16 rows stay on the decode family"); + check(wide_takes(17, 16, 32) && wide_takes(32, 16, 32), "wide takes: 17 and 32 rows go to the wide family"); + check(!wide_takes(33, 16, 32), "wide takes: 33 rows are not routed to a 32-row kernel"); + check(!wide_takes(17, 16, 0) && !wide_takes(17, 16, 16), "wide takes: no family, or one no wider than decode, takes nothing"); + std::printf(" wide family: 14 arms (complete or not, chunk width, which launch takes it)\n"); +} + int main(int argc, char** argv) { batch_width_arms(); + wide_family_arms(); quant_int4_arms(); for (int G : {8, 9, 16}) { for (int n : {G, 3 * G}) { // whole groups: a ragged tail is not a permutation From f1aa6e4d50303759ed18cc1aa7a3f0fb442e42f6 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 18:33:10 -0400 Subject: [PATCH 2/2] npu: a pack can name its wide sealed set (q4nx__sealed_wide_elf); the environment overrides or switches it off (Refs #1990) config.json keys q4nx_t2s_sealed_wide_elf / q4nx_pb16i_sealed_wide_elf (a file in the kernels dir) and q4nx_sealed_wide_rows (default 32). NPU_WIDE_SEALED_ELF= overrides, =0 switches the pack's family off. A named file that is missing or does not load is logged and the pass stays at 16 rows. wide_elf_path() holds the choice and has 7 arms in tests/npu_pack_test.cpp. Measured through the pack key on bonsai2-27b-t2s2: first token 28.7-29.0 tok/s off, 33.9-34.4 tok/s on, same text, oracle MATCH. Co-Authored-By: Claude Fable 5.2 --- docs/npu.md | 45 +++++++++++++++++++----------- npu/forward/include/batch_width.h | 13 +++++++++ npu/forward/include/q4nx_forward.h | 3 ++ npu/forward/src/q4nx_forward.cpp | 26 +++++++++++------ tests/npu_pack_test.cpp | 11 +++++++- 5 files changed, 72 insertions(+), 26 deletions(-) diff --git a/docs/npu.md b/docs/npu.md index 8332eae33..08d054047 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -538,23 +538,34 @@ second pair is not quoted as a speedup. `NPU_DEQ_LDEXP=2` runs both forms and co `tests/npu_pb16i_layout_test.cpp` holds the two forms to the same bits for one and two planes, the edges of the exponent range and exponents outside it. The `[bl]` line names the term as `deq`. -**A sealed set may carry a wide family for prefill (#1990, experimental, off by default).** The served -sealed sets are built for 16 rows per launch, so a prefill chunk is 16 rows and a 536-token prompt is 34 -blocks of 416 launches. `NPU_WIDE_SEALED_ELF=` names a second sealed ELF with the same instances -built for `NPU_WIDE_ROWS` rows (default 32, allowed 17..128). When it loads for every role that has a -16-row kernel, the chunk width becomes that row count and a launch of more than 16 rows takes the wide -kernel; decode steps and chunks of up to 16 rows stay on the 16-row family. It is all roles or none: the -chunk width is one number for the pass, so a set missing one role is not adopted and the serve log says -why (`[forward] wide family NOT adopted (...)`). The weight images are shared, only the activation and -result buffers differ. A launch that carries more rows than its kernel holds is refused by name. The rules -(`wide_family_complete`, `chunk_rows`, `wide_takes` in `npu/forward/include/batch_width.h`) are pinned -without a device by `tests/npu_pack_test.cpp`. No pack ships a wide ELF and the plan has no key for one -yet, which is why this is an environment switch. Measured on `bonsai2-27b-t2s2` with a 32-row set, one -`npu-hold` ticket, one binary, arms default / wide / default / wide: three 510-536-token prompts reached their first token at 28.3-29.8 and 30.5-30.8 tok/s -on the default path and **35.6-35.9 and 35.0-35.4 tok/s** with the wide set. The block median was -534.2 and 509.3 ms for 16 rows (103 blocks per arm) against 871.0 and 881.5 ms for 32 rows (50 blocks per -arm): 30.0-31.4 rows/s -> **36.3-36.7 rows/s**. The 8-token greedy text is identical in every arm, and `NPU_BATCH_CHECK=1` on the wide -path reports MATCH on every chunk it checks (32-, 4- and 1-row). +**A sealed set may carry a wide family for prefill (#1990).** The served sealed sets are built for 16 +rows per launch, so a prefill chunk is 16 rows and a 536-token prompt is 34 blocks of 416 launches. A pack +may name a second sealed ELF with the same instances built for more rows: `q4nx_t2s_sealed_wide_elf` (or +`q4nx_pb16i_sealed_wide_elf`) in `config.json`, a file in the kernels directory beside the 16-row one, and +`q4nx_sealed_wide_rows` for its row count (default 32, allowed 17..128). No pack ships one yet, so nothing +changes for an existing pack. `NPU_WIDE_SEALED_ELF=` overrides the pack's file, `NPU_WIDE_SEALED_ELF=0` +switches a pack's wide family off, and `NPU_WIDE_ROWS` overrides the row count. + +When the wide ELF loads for every role that has a 16-row kernel, the chunk width becomes its row count and a +launch of more than 16 rows takes the wide kernel; decode steps and chunks of up to 16 rows stay on the +16-row family. It is all roles or none: the chunk width is one number for the pass, so a set missing one +role is not adopted. A named file that is missing or does not load never stops the serve: the log says why +(`[forward] wide family NOT adopted (...)`) and chunks stay at 16 rows. The weight images are shared, only +the activation and result buffers differ. A launch that carries more rows than its kernel holds is refused +by name. The rules (`wide_family_complete`, `chunk_rows`, `wide_takes`, `wide_elf_path` in +`npu/forward/include/batch_width.h`) are pinned without a device by `tests/npu_pack_test.cpp`. + +Measured on `bonsai2-27b-t2s2` with a 32-row set, one `npu-hold` ticket per row of the table, one binary, +arms off / on / off / on, three 510-536-token prompts per arm (`prompt_tokens / wall`, `max_tokens 1`): + +| how the wide set was named | first token, wide off | first token, wide on | 16-row block median | 32-row block median | +|---|---|---|---|---| +| `NPU_WIDE_SEALED_ELF` | 28.3-29.8, 30.5-30.8 tok/s | **35.6-35.9, 35.0-35.4 tok/s** | 534.2, 509.3 ms | 871.0, 881.5 ms | +| the pack's `config.json` key | 28.7-28.9, 28.7-29.0 tok/s | **33.9-34.2, 34.2-34.4 tok/s** | 541.2, 541.4 ms | 914.0, 908.3 ms | + +Two 16-row blocks against one 32-row block is 30-31 rows/s against 35-37. The 8-token greedy text is +identical with the family off and on, and `NPU_BATCH_CHECK=1` on the wide path reports MATCH on every chunk +it checks (32-, 4- and 1-row) in both tickets. Only 32 rows has been measured. **A t2s container wants a sealed set for the same reason a pb16i one does.** The layout is not what makes a phase change expensive: the **hw_context** is. Measured on this box in-loop on 2026-11-02, a t2s decode diff --git a/npu/forward/include/batch_width.h b/npu/forward/include/batch_width.h index 3ad110da9..3365c7ff0 100644 --- a/npu/forward/include/batch_width.h +++ b/npu/forward/include/batch_width.h @@ -25,6 +25,8 @@ // pinned by a host test with no device, no ELF and no runtime. #pragma once +#include + namespace onebit { namespace npu { @@ -73,5 +75,16 @@ inline int chunk_rows(int narrow, int wide) { return wide > narrow ? wide : narr // launch past `wide` rows is never routed here (the caller refuses it by name). inline bool wide_takes(int rows, int narrow, int wide) { return wide > narrow && rows > narrow && rows <= wide; } +// Which file is the wide sealed ELF, if any? The environment wins over the pack so a set can be tried, or switched +// off, without editing a pack: `env` "0" means no wide family even when the pack names one, any other non-empty +// value is the path as given. Otherwise the pack's name is used: as given when absolute, else inside `kernels_dir` +// (where the 16-row sealed ELF lives). Empty answer = no wide family. `env` may be null. +inline std::string wide_elf_path(const char* env, const std::string& pack_name, const std::string& kernels_dir) { + if (env && *env) return (env[0] == '0' && env[1] == 0) ? std::string() : std::string(env); + if (pack_name.empty()) return std::string(); + if (pack_name[0] == '/' || kernels_dir.empty()) return pack_name; + return kernels_dir + "/" + pack_name; +} + } // namespace npu } // namespace onebit diff --git a/npu/forward/include/q4nx_forward.h b/npu/forward/include/q4nx_forward.h index 403481ce6..4770d9969 100644 --- a/npu/forward/include/q4nx_forward.h +++ b/npu/forward/include/q4nx_forward.h @@ -364,6 +364,9 @@ class Q4nxNpuForward { // hw_contexts. Config `q4nx_pb16i_sealed_elf` names it in the kernels dir; without // it the per-role `_pb16i.elf` files are used as before. std::string pb16i_sealed_; + std::string wide_sealed_; // q4nx__sealed_wide_elf: a wide prefill family (engine#1990) + int wide_sealed_rows_ = 32; // q4nx_sealed_wide_rows + std::string sealed_dir_; // the kernels dir the 16-row sealed ELF was found in // ---- the NPU target (engine#799 P4) ---- // Everything that talks to a device lives behind this: the ELF contexts (M=128 prefill and the M=16 // decode twin), the phase-selected launch, the packed-weight BO cache, the pb16i sidecar images, the diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 4ee9fed39..5df0983fd 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -24,6 +24,7 @@ // Numerics follow tools/q4nx_forward_ref.py so the two can be compared // element-wise. #include "q4nx_forward.h" +#include "batch_width.h" #include "q4nx_state_planes.h" // the plane-copy half of a snapshot, device-free and unit-tested #include "model/op_dispatch.h" // the optional-op fallback seam (engine#803 A3) #include @@ -511,6 +512,10 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, // pb16i ELF (which would read 4,608-B tiles out of 2,176-B images). pb16i_sealed_ = read_config_string( cfg_path, t2s_ ? "\"q4nx_t2s_sealed_elf\"" : "\"q4nx_pb16i_sealed_elf\"", ""); + // engine#1990: an optional second sealed ELF, the same instances built for more rows per launch. + wide_sealed_ = read_config_string( + cfg_path, t2s_ ? "\"q4nx_t2s_sealed_wide_elf\"" : "\"q4nx_pb16i_sealed_wide_elf\"", ""); + wide_sealed_rows_ = q4nx_config_int(cfg_path.c_str(), "q4nx_sealed_wide_rows", 32); // A folded container also declares the fold: `q4nx_hadamard` names the sign table and // `q4nx_hadamard_block` the Hadamard block. Without it the sidecar images would be // read in the stored (rotated) basis against un-rotated activations. @@ -1342,6 +1347,7 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, continue; } elf = dir + "/" + pb16i_sealed_; + sealed_dir_ = dir; kname = inst; if (!file_exists(elf)) { err_ = plan_.loaded() ? ("plan " + plan_path_ + ": decode.sealed_elf \"" + @@ -1412,15 +1418,19 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, fprintf(stderr, " [forward] decode family: %d role(s) at M=%d, %d on M=%d%s\n", made, Q4NX_DESIGN_M_DECODE, skipped, Q4NX_DESIGN_M_PREFILL, want_m16 ? "" : " (NPU_DECODE_M16=0: disabled)"); - // engine#1990: a wide prefill family for a sealed pb16i/t2s set. NPU_WIDE_SEALED_ELF names a second sealed - // ELF carrying the same instances built for NPU_WIDE_ROWS rows (default 32); a chunk of more than 16 rows - // then runs one launch per role instead of two. Experimental and off unless the variable is set: no pack - // ships such an ELF yet, and the plan has no key for it. All roles or none -- max_batch_rows() is the chunk - // width, so one role without a wide kernel would be launched past its buffers. - if (const char* wide_elf = getenv("NPU_WIDE_SEALED_ELF"); - wide_elf && *wide_elf && pb16i_ && !pb16i_sealed_.empty() && made > 0) { + // engine#1990: a wide prefill family for a sealed pb16i/t2s set: a second sealed ELF carrying the same + // instances built for more rows per launch (default 32), so a chunk of more than 16 rows runs one launch per + // role instead of two. The pack names it (config `q4nx__sealed_wide_elf`, rows in + // `q4nx_sealed_wide_rows`); NPU_WIDE_SEALED_ELF overrides the file ("0" = off) and NPU_WIDE_ROWS the rows. + // All roles or none -- max_batch_rows() is the chunk width, so one role without a wide kernel would be + // launched past its buffers. A named file that is missing or does not load is logged and the pass stays at + // 16 rows: a pack must never stop serving over an optional speed-up. + const std::string wide_path = + onebit::npu::wide_elf_path(getenv("NPU_WIDE_SEALED_ELF"), wide_sealed_, sealed_dir_); + if (const char* wide_elf = wide_path.c_str(); + !wide_path.empty() && pb16i_ && !pb16i_sealed_.empty() && made > 0) { const char* wr = getenv("NPU_WIDE_ROWS"); - const int rows = wr && *wr ? atoi(wr) : 32; + const int rows = wr && *wr ? atoi(wr) : wide_sealed_rows_; int wide = 0; std::string why; if (rows <= Q4NX_DESIGN_M_DECODE || rows > Q4NX_DESIGN_M_PREFILL) why = "NPU_WIDE_ROWS must be 17..128"; diff --git a/tests/npu_pack_test.cpp b/tests/npu_pack_test.cpp index 16d508fda..879e43ec6 100644 --- a/tests/npu_pack_test.cpp +++ b/tests/npu_pack_test.cpp @@ -141,7 +141,16 @@ static void wide_family_arms() { check(wide_takes(17, 16, 32) && wide_takes(32, 16, 32), "wide takes: 17 and 32 rows go to the wide family"); check(!wide_takes(33, 16, 32), "wide takes: 33 rows are not routed to a 32-row kernel"); check(!wide_takes(17, 16, 0) && !wide_takes(17, 16, 16), "wide takes: no family, or one no wider than decode, takes nothing"); - std::printf(" wide family: 14 arms (complete or not, chunk width, which launch takes it)\n"); + using onebit::npu::wide_elf_path; + check(wide_elf_path(nullptr, "", "/k").empty() && wide_elf_path("", "", "/k").empty(), + "wide elf: neither the environment nor the pack names one -> none"); + check(wide_elf_path(nullptr, "W.elf", "/k") == "/k/W.elf", "wide elf: the pack's name is looked up in the kernels dir"); + check(wide_elf_path("", "W.elf", "/k") == "/k/W.elf", "wide elf: an empty variable is not an override"); + check(wide_elf_path(nullptr, "/abs/W.elf", "/k") == "/abs/W.elf", "wide elf: an absolute pack name is used as given"); + check(wide_elf_path("/e/X.elf", "W.elf", "/k") == "/e/X.elf", "wide elf: the environment wins over the pack"); + check(wide_elf_path("0", "W.elf", "/k").empty(), "wide elf: NPU_WIDE_SEALED_ELF=0 switches a pack's wide family off"); + check(wide_elf_path("0x.elf", "", "/k") == "0x.elf", "wide elf: only the exact value 0 means off"); + std::printf(" wide family: 21 arms (complete or not, chunk width, which launch takes it, which file)\n"); } int main(int argc, char** argv) {