From b9ce304ddee89764c2eda445eceeee330bca48bd Mon Sep 17 00:00:00 2001 From: t Date: Tue, 10 Nov 2026 07:23:08 -0400 Subject: [PATCH 1/3] npu: the sealed route reports its design's batch width instead of 1, and the aggregate is the MINIMUM (engine#1900) `max_rows()` looped over the per-role `ctx_` / `ctx16_` arrays and returned the first populated entry. The **sealed** route keeps its design in one ELF (`pb16i_sealed_`) and populates neither array, so both loops found nothing and every sealed pack reported width **1**: 1bit npu-run: 7 ids, so this runs in passes of 1 rows (the kernel's width) with the plan's own widths at 16/128 and the ELF compiled for them. The chunker then walked N single-row passes -- the whole point of `step_batch`'s chunking -- so #1781 gate 1 ("rows 2..k independent of row 1 within one pass") had no pack on this box to measure on, and nothing in the log said why. Two changes, both specified by the issue: * **the sealed fallback.** When no per-role context is populated and a sealed ELF is set, the width is the **loaded design's own M**, which `caps()` already answers (decode twin vs prefill) -- not a guess at `setup_.M`, and not the prefill width when the decode design is the one loaded. * **min, not first.** `batch_width_min()` (new `npu/forward/include/batch_width.h`, pure and header-only) takes the SMALLEST positive width. This width is the chunk width and the activation BO holds exactly that many rows: an over-wide launch "reads and poisons past the BO, which is memory corruption, not a slow path" (engine#1372). The old first-populated answer returned **128 for a 16/16/128 set** whenever the wide role happened to be index 0. The width is now printed once at load -- `[forward] batch width N row(s) per launch` -- because the failure mode was a log that looked normal while nothing batched. Arms (7, host-only, in `tests/npu_pack_test.cpp`): 16/16/128 answers 16; **128 at index 0 does not win** (the old behaviour, and the corruption direction); unloaded zero slots are skipped; an empty array takes the fallback; a null array and `n=0` read nothing. Proved they can fail by restoring the first-populated rule: 2 arms report FAIL, revert is green. Suite 176/176; copyright clean. Refs #1900, #1862, #1781, #1372. --- npu/forward/include/batch_width.h | 46 +++++++++++++++++++++++++++++++ npu/forward/src/q4nx_forward.cpp | 4 +++ npu/target/xdna/xdna_target.cpp | 25 +++++++++++++++-- tests/npu_pack_test.cpp | 24 ++++++++++++++++ 4 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 npu/forward/include/batch_width.h diff --git a/npu/forward/include/batch_width.h b/npu/forward/include/batch_width.h new file mode 100644 index 000000000..9901849a9 --- /dev/null +++ b/npu/forward/include/batch_width.h @@ -0,0 +1,46 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// batch_width.h -- how many rows one launch may carry (engine#1900). +// +// The forward chunks a batched pass at this width, and the activation BO holds exactly this many rows: an over-wide +// launch reads and poisons past the BO, which is memory corruption rather than a slow path (engine#1372). So the +// aggregate over a design set must be the MINIMUM of the widths actually loaded, never the first one that happens to +// be populated -- a 16/16/128 set returning 128 corrupts memory on the two narrow roles, while returning 16 is only +// narrower than it needs to be. +// +// Pure and header-only on purpose: the aggregation is the part that can be wrong in the dangerous direction, so it is +// pinned by a host test with no device, no ELF and no runtime. +#pragma once + +namespace onebit { +namespace npu { + +// The smallest positive width in `widths`, or `fallback` when none is positive. +// +// `widths` may contain 0 for "no design at this role", which is the ordinary state of a design array -- the old code +// returned the FIRST populated entry, so a zero-width slot simply moved the answer to whichever role came next. +inline int batch_width_min(const int* widths, int n, int fallback) { + int w = 0; + if (widths) + for (int i = 0; i < n; ++i) { + const int v = widths[i]; + if (v > 0 && (w == 0 || v < w)) w = v; + } + return w > 0 ? w : (fallback > 0 ? fallback : 1); +} + +} // namespace npu +} // namespace onebit diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index a8e503162..5200c8983 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -1545,6 +1545,10 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, backend_ = "full ELF"; ready_ = true; + /* engine#1900: say the batch width once at load, the way the block note does, so a log can be told apart from a + * forward that never batched. Before this, a sealed pack reported 1 through the whole run and nothing said so -- + * the only symptom was N single-row passes and #1781 gate 1 having no pack to measure on. */ + std::fprintf(stderr, " [forward] batch width %d row(s) per launch\n", max_batch_rows()); return true; } diff --git a/npu/target/xdna/xdna_target.cpp b/npu/target/xdna/xdna_target.cpp index fe4c4ecae..d3218d1d6 100644 --- a/npu/target/xdna/xdna_target.cpp +++ b/npu/target/xdna/xdna_target.cpp @@ -20,6 +20,7 @@ // // docs/npu-p4-target-split.md records what moved in A1 and what A2 (plan-driven setup) still owes. #include "target/xdna/xdna_target.h" +#include "batch_width.h" #include #include @@ -281,8 +282,28 @@ bool XdnaTarget::weight_cached(int layer, int key) const { } int XdnaTarget::max_rows() const { - if (pb16i_) { for (int i = 0; i < ndesigns_; i++) if (ctx16_[i]) return ctx16_[i]->MD; } - for (int i = 0; i < ndesigns_; i++) if (ctx_[i]) return ctx_[i]->MD; + // engine#1900: the SEALED route keeps its design in one ELF and populates no per-role ctx_, so both loops used to + // find nothing and every sealed pack reported width 1 -- the chunker then walked N single-row passes, which is + // the whole point of step_batch's chunking. The width is known: the sealed ELF is plan-derived and `caps()` + // already answers which design is loaded (the decode twin or the prefill one). + // + // MIN over the loaded widths, never the first populated one: this width is the chunk width, and an over-wide + // launch reads past the activation BO (engine#1372 -- memory corruption, not a slow path). A 16/16/128 set must + // therefore answer 16; answering 128 because that role happens to be index 0 is the dangerous direction. + const int n = ndesigns_; + int w = 0; + if (pb16i_) { + int widths[Q4NX_MAX_DESIGNS] = {0}; + for (int i = 0; i < n && i < Q4NX_MAX_DESIGNS; i++) widths[i] = ctx16_[i] ? ctx16_[i]->MD : 0; + w = onebit::npu::batch_width_min(widths, n < Q4NX_MAX_DESIGNS ? n : Q4NX_MAX_DESIGNS, 1); + if (w <= 1 && !pb16i_sealed_.empty()) w = caps().max_rows; // sealed: no ctx, the loaded design's own M + if (w > 1) return w; + } else { + int widths[Q4NX_MAX_DESIGNS] = {0}; + for (int i = 0; i < n && i < Q4NX_MAX_DESIGNS; i++) widths[i] = ctx_[i] ? ctx_[i]->MD : 0; + w = onebit::npu::batch_width_min(widths, n < Q4NX_MAX_DESIGNS ? n : Q4NX_MAX_DESIGNS, 1); + if (w > 1) return w; + } return 1; } diff --git a/tests/npu_pack_test.cpp b/tests/npu_pack_test.cpp index a544d9601..35e3d5805 100644 --- a/tests/npu_pack_test.cpp +++ b/tests/npu_pack_test.cpp @@ -23,6 +23,7 @@ #include "full_elf.h" #include "pack.h" #include "q4nx_pack.h" +#include "batch_width.h" #include "golden/host_gemm_i8_fixture.h" @@ -86,7 +87,30 @@ static void quant_int4_arms() { std::printf(" int4 pair: 11 arms (known codes, ladder, floor, NaN, round-trip)\n"); } +/* engine#1900: the batch width is the chunk width, and the activation BO holds exactly that many rows -- an OVER-wide + * answer reads past the BO (engine#1372, memory corruption rather than a slow path). The old code returned the first + * populated design's width, so a 16/16/128 set answered 128 whenever that role was index 0. The arms below are the + * dangerous direction (min, not first), the empty array (sealed packs: no per-role ctx at all, hence a fallback), and + * the zero slots that an unloaded role leaves behind. */ +static void batch_width_arms() { + using onebit::npu::batch_width_min; + const int mixed[] = {16, 16, 128}; + check(batch_width_min(mixed, 3, 1) == 16, "batch width: 16/16/128 answers 16, not the 128 at index 2"); + const int wide_first[] = {128, 16, 16}; + check(batch_width_min(wide_first, 3, 1) == 16, + "batch width: 128 at index 0 does not win (the old code returned the FIRST populated entry)"); + const int slots[] = {0, 0, 16, 0}; + check(batch_width_min(slots, 4, 1) == 16, "batch width: unloaded slots are skipped, not treated as a width"); + const int none[] = {0, 0, 0}; + check(batch_width_min(none, 3, 1) == 1, "batch width: no design at all falls back to 1 (a single-row launch)"); + check(batch_width_min(none, 3, 16) == 16, "batch width: the fallback is the loaded design's own M when the caller has it"); + check(batch_width_min(nullptr, 0, 128) == 128, "batch width: a null array reads as empty and takes the fallback"); + check(batch_width_min(mixed, 0, 4) == 4, "batch width: an n of 0 reads nothing and takes the fallback"); + std::printf(" batch width: 7 arms (min not first, empty slots, fallback)\n"); +} + int main(int argc, char** argv) { + batch_width_arms(); quant_int4_arms(); for (int G : {8, 9, 16}) { for (int n : {G, 3 * G}) { // whole groups: a ragged tail is not a permutation From 8eec2a8a6f55fc3ba54a93a68c39ac199bb3e426 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 10 Nov 2026 07:47:45 -0400 Subject: [PATCH 2/3] npu: the batch width is the minimum loaded design width, not the first (engine#1900 review) Review of #1903 was right on the blocking point, and I am not keeping the half it rejected. The sealed fallback I wrote went through `caps()`: c.max_rows = m16_ ? Q4NX_DESIGN_M_DECODE : Q4NX_DESIGN_M_PREFILL; and `m16_` is set only when 16-row per-role contexts are created -- which the sealed route does not do, the premise of #1900. So for every sealed pack that returned the compile-time `Q4NX_DESIGN_M_PREFILL` (128) whatever the pack's plan or ELF was built at: a guess at which design is loaded, which is exactly what the issue says not to do, and on a sealed set compiled at a smaller M it would launch more rows than the activation BO holds (engine#1372). That half belongs to **#1904**, which reads the pack's own `plan_.prefill_m` and measured the result on the device -- one pass of 4 rows, `identical=4 differing=0`. What is left here is the part #1904 does not have: **the aggregate is the MINIMUM of the loaded per-role widths.** This width is the chunk width and the BO holds exactly that many rows, so an over-wide answer is memory corruption (engine#1372). The old code returned the FIRST populated entry, so a 16/16/128 set answered 128 whenever the wide role happened to be index 0 -- the dangerous direction was live even on the array path the issue treats as fine. `batch_width_min()` (pure, header-only, `npu/forward/include/batch_width.h`) is the smallest positive width. The load line now says which case it is: `batch width N row(s) per launch`, with an explicit "(single row: no per-role design width -- the sealed route reports its plan's width in engine#1904)" when the answer is 1, so a sealed pack's log does not look like a batched run that chose 1. Arms unchanged (7, host-only), including `128 at index 0 does not win`; restoring the first-populated rule still makes 2 arms fail. Suite 176/176. Refs #1900, #1904, #1372. --- npu/forward/src/q4nx_forward.cpp | 4 +++- npu/target/xdna/xdna_target.cpp | 40 ++++++++++++++------------------ 2 files changed, 20 insertions(+), 24 deletions(-) diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 5200c8983..d78289178 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -1548,7 +1548,9 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, /* engine#1900: say the batch width once at load, the way the block note does, so a log can be told apart from a * forward that never batched. Before this, a sealed pack reported 1 through the whole run and nothing said so -- * the only symptom was N single-row passes and #1781 gate 1 having no pack to measure on. */ - std::fprintf(stderr, " [forward] batch width %d row(s) per launch\n", max_batch_rows()); + std::fprintf(stderr, " [forward] batch width %d row(s) per launch%s\n", max_batch_rows(), + max_batch_rows() <= 1 ? " (single row: no per-role design width -- the sealed route reports its plan's " + "width in engine#1904)" : ""); return true; } diff --git a/npu/target/xdna/xdna_target.cpp b/npu/target/xdna/xdna_target.cpp index d3218d1d6..5956a5403 100644 --- a/npu/target/xdna/xdna_target.cpp +++ b/npu/target/xdna/xdna_target.cpp @@ -282,29 +282,23 @@ bool XdnaTarget::weight_cached(int layer, int key) const { } int XdnaTarget::max_rows() const { - // engine#1900: the SEALED route keeps its design in one ELF and populates no per-role ctx_, so both loops used to - // find nothing and every sealed pack reported width 1 -- the chunker then walked N single-row passes, which is - // the whole point of step_batch's chunking. The width is known: the sealed ELF is plan-derived and `caps()` - // already answers which design is loaded (the decode twin or the prefill one). - // - // MIN over the loaded widths, never the first populated one: this width is the chunk width, and an over-wide - // launch reads past the activation BO (engine#1372 -- memory corruption, not a slow path). A 16/16/128 set must - // therefore answer 16; answering 128 because that role happens to be index 0 is the dangerous direction. - const int n = ndesigns_; - int w = 0; - if (pb16i_) { - int widths[Q4NX_MAX_DESIGNS] = {0}; - for (int i = 0; i < n && i < Q4NX_MAX_DESIGNS; i++) widths[i] = ctx16_[i] ? ctx16_[i]->MD : 0; - w = onebit::npu::batch_width_min(widths, n < Q4NX_MAX_DESIGNS ? n : Q4NX_MAX_DESIGNS, 1); - if (w <= 1 && !pb16i_sealed_.empty()) w = caps().max_rows; // sealed: no ctx, the loaded design's own M - if (w > 1) return w; - } else { - int widths[Q4NX_MAX_DESIGNS] = {0}; - for (int i = 0; i < n && i < Q4NX_MAX_DESIGNS; i++) widths[i] = ctx_[i] ? ctx_[i]->MD : 0; - w = onebit::npu::batch_width_min(widths, n < Q4NX_MAX_DESIGNS ? n : Q4NX_MAX_DESIGNS, 1); - if (w > 1) return w; - } - return 1; + /* engine#1900 (review of #1903): this width IS the chunk width, and the activation BO holds exactly that many rows -- + * an over-wide launch reads and poisons past the BO, which is memory corruption rather than a slow path (engine#1372). + * The old code returned the FIRST populated per-role entry, so a 16/16/128 set answered 128 whenever the wide role + * happened to be index 0: the dangerous direction was live even on the array path. + * + * The MINIMUM of the loaded widths is therefore the only safe aggregate. The SEALED-wide case (no per-role context is + * populated at all, so this falls through) is NOT decided here: the sealed branch took the width from a compile-time + * constant through caps(), which is a guess at which design is loaded, and the pack's own statement is `plan_m` -- see + * #1904, which reads it and measured one 4-row pass on the device (`identical=4 differing=0`). Whichever of the two + * lands first, this function must not answer wider than the narrowest loaded design. */ + const int n = ndesigns_ < Q4NX_MAX_DESIGNS ? ndesigns_ : Q4NX_MAX_DESIGNS; + int widths[Q4NX_MAX_DESIGNS] = {0}; + if (pb16i_) + for (int i = 0; i < n; i++) widths[i] = ctx16_[i] ? ctx16_[i]->MD : 0; + else + for (int i = 0; i < n; i++) widths[i] = ctx_[i] ? ctx_[i]->MD : 0; + return onebit::npu::batch_width_min(widths, n, 1); } int XdnaTarget::rows_for(int role, int want) const { From cf537816f2187cbc550da052bc3c05c3d5d61cc6 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 10 Nov 2026 08:06:46 -0400 Subject: [PATCH 3/3] npu: keep the pb16i fall-through to the prefill contexts (engine#1900 review, second pass) Second review was right again: my rewrite turned main's two loops into an `if / else`, so with `pb16i_` set only the `ctx16_` array was read. A pb16i set that adopted only its prefill kernels has no 16-row twin, so `batch_width_min` got all zeros and answered **1** -- the #1900 symptom ("passes of 1 rows") on a different route, where main batched at 128. Main's shape is a fall-through, and this restores it while keeping the minimum: if any ctx16_[i] -> min over the 16-row twins (a pb16i set that made them) else -> min over ctx_[i] (the prefill set, or a non-pb16i build) else -> 1 The arm comment is also corrected on the reviewer's non-blocking point: these arms pin the RULE (minimise, skip empty slots, fall back), and no pack on this box has been shown to adopt mixed widths -- so nothing here claims the old answer was wrong on a real pack, only that it would be the dangerous way round if one ever did. The sealed route's width is #1904's, read from the plan and measured on the device. Suite 176/176. Refs #1900, #1904, #1372. --- npu/target/xdna/xdna_target.cpp | 31 ++++++++++++++++++------------- tests/npu_pack_test.cpp | 9 ++++++--- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/npu/target/xdna/xdna_target.cpp b/npu/target/xdna/xdna_target.cpp index 5956a5403..cd892f7fb 100644 --- a/npu/target/xdna/xdna_target.cpp +++ b/npu/target/xdna/xdna_target.cpp @@ -282,22 +282,27 @@ bool XdnaTarget::weight_cached(int layer, int key) const { } int XdnaTarget::max_rows() const { - /* engine#1900 (review of #1903): this width IS the chunk width, and the activation BO holds exactly that many rows -- - * an over-wide launch reads and poisons past the BO, which is memory corruption rather than a slow path (engine#1372). - * The old code returned the FIRST populated per-role entry, so a 16/16/128 set answered 128 whenever the wide role - * happened to be index 0: the dangerous direction was live even on the array path. + /* engine#1900: this width IS the chunk width, and the activation BO holds exactly that many rows -- an over-wide + * launch reads and poisons past the BO, which is memory corruption rather than a slow path (engine#1372). The old + * code returned the FIRST populated per-role entry, so a mixed set answered whichever role happened to be index 0. * - * The MINIMUM of the loaded widths is therefore the only safe aggregate. The SEALED-wide case (no per-role context is - * populated at all, so this falls through) is NOT decided here: the sealed branch took the width from a compile-time - * constant through caps(), which is a guess at which design is loaded, and the pack's own statement is `plan_m` -- see - * #1904, which reads it and measured one 4-row pass on the device (`identical=4 differing=0`). Whichever of the two - * lands first, this function must not answer wider than the narrowest loaded design. */ + * The MINIMUM of the loaded widths is the only safe aggregate, and the pb16i_ case keeps main's FALL-THROUGH: a + * pb16i set that adopted only its prefill kernels has no 16-row twin, and answering 1 there would walk single-row + * passes (the #1900 symptom on a different route). + * + * The sealed route (no per-role context at all) is NOT decided here: the pack's own statement is `plan_m`, which + * #1904 reads and measured on the device. Whichever lands first, this must never answer wider than the narrowest + * loaded design. */ const int n = ndesigns_ < Q4NX_MAX_DESIGNS ? ndesigns_ : Q4NX_MAX_DESIGNS; - int widths[Q4NX_MAX_DESIGNS] = {0}; + int widths16[Q4NX_MAX_DESIGNS] = {0}; + bool any16 = false; if (pb16i_) - for (int i = 0; i < n; i++) widths[i] = ctx16_[i] ? ctx16_[i]->MD : 0; - else - for (int i = 0; i < n; i++) widths[i] = ctx_[i] ? ctx_[i]->MD : 0; + for (int i = 0; i < n; i++) + if (ctx16_[i]) { widths16[i] = ctx16_[i]->MD; any16 = true; } + if (any16) return onebit::npu::batch_width_min(widths16, n, 1); + int widths[Q4NX_MAX_DESIGNS] = {0}; + for (int i = 0; i < n; i++) + if (ctx_[i]) widths[i] = ctx_[i]->MD; return onebit::npu::batch_width_min(widths, n, 1); } diff --git a/tests/npu_pack_test.cpp b/tests/npu_pack_test.cpp index 35e3d5805..e8f354a95 100644 --- a/tests/npu_pack_test.cpp +++ b/tests/npu_pack_test.cpp @@ -89,9 +89,12 @@ static void quant_int4_arms() { /* engine#1900: the batch width is the chunk width, and the activation BO holds exactly that many rows -- an OVER-wide * answer reads past the BO (engine#1372, memory corruption rather than a slow path). The old code returned the first - * populated design's width, so a 16/16/128 set answered 128 whenever that role was index 0. The arms below are the - * dangerous direction (min, not first), the empty array (sealed packs: no per-role ctx at all, hence a fallback), and - * the zero slots that an unloaded role leaves behind. */ + * populated design's width, so a mixed set answered whichever role was index 0. + * + * These arms pin the RULE (minimise, skip empty slots, take a fallback), not a measurement: no pack on this box has been + * shown to adopt mixed widths, so nothing here claims the old answer was wrong on a real pack -- only that it would be + * the dangerous way round if one ever did. The sealed route's width is #1904's, read from the plan and measured on the + * device. */ static void batch_width_arms() { using onebit::npu::batch_width_min; const int mixed[] = {16, 16, 128};