From 84fabb75d4a29435a5cb5c1a892663e6ba968ee5 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 10 Nov 2026 11:00:58 -0400 Subject: [PATCH] npu: the batch width is the minimum loaded width, on top of #1904's plan-derived sealed width (engine#1900 review) Rebased onto `main` after **#1904 merged**, because my branch as written deleted its half: #1904 gives the sealed route its width from `plan_.prefill_m` (the pack's own statement, device-measured at one 4-row pass), and my `max_rows()` rewrite replaced the body wholesale. This keeps both properties: * **#1904's plan-derived width survives untouched** at the end of the function, for the sealed/mono-ELF generation that populates neither per-role array; * **the aggregate over the loaded per-role widths is the MINIMUM**, not the first populated entry. This width is the chunk width and the activation BO holds exactly that many rows: an over-wide answer reads and poisons past the BO, which is memory corruption rather than a slow path (engine#1372). Returning the FIRST entry meant a mixed set answered whichever role happened to be index 0, so a 16/16/128 set could answer 128 -- the dangerous direction; * **the pb16i_ fall-through is main's**: a pb16i set that adopted only its prefill kernels has no 16-row twin, and answering 1 there would walk single-row passes (the #1900 symptom on a different route). `batch_width_min()` (pure, header-only) is unchanged, with its 7 arms -- including `128 at index 0 does not win`, which fails on the old first-populated rule. The load line still prints the width once: `[forward] batch width N row(s) per launch`. Suite 182/182 on this rebase. The one failure I hit first was `batch_check_state` "Not Run" -- a target #1919 added that my incremental build had not built, not a regression; building it cleared the run to 182/182. Refs #1900, #1904, #1903, #1372. --- npu/forward/include/batch_width.h | 46 +++++++++++++++++++++++++++++++ npu/forward/src/q4nx_forward.cpp | 4 +++ npu/target/xdna/xdna_target.cpp | 22 +++++++++++++-- tests/npu_pack_test.cpp | 27 ++++++++++++++++++ 4 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 npu/forward/include/batch_width.h diff --git a/npu/forward/include/batch_width.h b/npu/forward/include/batch_width.h new file mode 100644 index 000000000..9901849a9 --- /dev/null +++ b/npu/forward/include/batch_width.h @@ -0,0 +1,46 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// batch_width.h -- how many rows one launch may carry (engine#1900). +// +// The forward chunks a batched pass at this width, and the activation BO holds exactly this many rows: an over-wide +// launch reads and poisons past the BO, which is memory corruption rather than a slow path (engine#1372). So the +// aggregate over a design set must be the MINIMUM of the widths actually loaded, never the first one that happens to +// be populated -- a 16/16/128 set returning 128 corrupts memory on the two narrow roles, while returning 16 is only +// narrower than it needs to be. +// +// Pure and header-only on purpose: the aggregation is the part that can be wrong in the dangerous direction, so it is +// pinned by a host test with no device, no ELF and no runtime. +#pragma once + +namespace onebit { +namespace npu { + +// The smallest positive width in `widths`, or `fallback` when none is positive. +// +// `widths` may contain 0 for "no design at this role", which is the ordinary state of a design array -- the old code +// returned the FIRST populated entry, so a zero-width slot simply moved the answer to whichever role came next. +inline int batch_width_min(const int* widths, int n, int fallback) { + int w = 0; + if (widths) + for (int i = 0; i < n; ++i) { + const int v = widths[i]; + if (v > 0 && (w == 0 || v < w)) w = v; + } + return w > 0 ? w : (fallback > 0 ? fallback : 1); +} + +} // namespace npu +} // namespace onebit diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index b587a1eb6..aa411398f 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -1545,6 +1545,10 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, backend_ = "full ELF"; ready_ = true; + /* engine#1900: say the batch width once at load, the way the block note does, so a log can be told apart from a + * forward that never batched. #1904 gives the sealed route its plan's width; this line is what makes the difference + * visible either way. */ + std::fprintf(stderr, " [forward] batch width %d row(s) per launch\n", max_batch_rows()); return true; } diff --git a/npu/target/xdna/xdna_target.cpp b/npu/target/xdna/xdna_target.cpp index 5a3621a93..3c6dcdc45 100644 --- a/npu/target/xdna/xdna_target.cpp +++ b/npu/target/xdna/xdna_target.cpp @@ -20,6 +20,7 @@ // // docs/npu-p4-target-split.md records what moved in A1 and what A2 (plan-driven setup) still owes. #include "target/xdna/xdna_target.h" +#include "batch_width.h" #include #include @@ -281,8 +282,25 @@ bool XdnaTarget::weight_cached(int layer, int key) const { } int XdnaTarget::max_rows() const { - if (pb16i_) { for (int i = 0; i < ndesigns_; i++) if (ctx16_[i]) return ctx16_[i]->MD; } - for (int i = 0; i < ndesigns_; i++) if (ctx_[i]) return ctx_[i]->MD; + /* engine#1900/#1903: MIN over the loaded per-role widths, never the first populated one. This width IS the chunk + * width, and the activation BO holds exactly that many rows -- an over-wide launch reads and poisons past the BO, + * which is memory corruption rather than a slow path (engine#1372). Returning the FIRST entry meant a mixed set + * answered whichever role happened to be index 0, so a 16/16/128 set could answer 128: the dangerous direction. + * The pb16i_ case keeps main's fall-through (a pb16i set that adopted only its prefill kernels has no 16-row twin, + * and answering 1 there would walk single-row passes), and #1904's plan-derived width for the sealed route stays + * below exactly as merged. */ + const int n = ndesigns_ < Q4NX_MAX_DESIGNS ? ndesigns_ : Q4NX_MAX_DESIGNS; + int widths16[Q4NX_MAX_DESIGNS] = {0}; + bool any16 = false; + if (pb16i_) + for (int i = 0; i < n; i++) + if (ctx16_[i]) { widths16[i] = ctx16_[i]->MD; any16 = true; } + if (any16) return onebit::npu::batch_width_min(widths16, n, 1); + int widths[Q4NX_MAX_DESIGNS] = {0}; + bool any_ctx = false; + for (int i = 0; i < n; i++) + if (ctx_[i]) { widths[i] = ctx_[i]->MD; any_ctx = true; } + if (any_ctx) return onebit::npu::batch_width_min(widths, n, 1); // engine#1900: the sealed/mono-ELF generation adopts its kernel set through a path that fills neither array, so both // loops above find nothing and this returned 1. This is the CHUNK WIDTH (q4nx_forward's `blk = max_batch_rows()`), so // those packs batched ONE ROW per launch however wide their design is -- `models/npu/bonsai2-27b-NPU-b8` states prefill diff --git a/tests/npu_pack_test.cpp b/tests/npu_pack_test.cpp index a544d9601..e8f354a95 100644 --- a/tests/npu_pack_test.cpp +++ b/tests/npu_pack_test.cpp @@ -23,6 +23,7 @@ #include "full_elf.h" #include "pack.h" #include "q4nx_pack.h" +#include "batch_width.h" #include "golden/host_gemm_i8_fixture.h" @@ -86,7 +87,33 @@ static void quant_int4_arms() { std::printf(" int4 pair: 11 arms (known codes, ladder, floor, NaN, round-trip)\n"); } +/* engine#1900: the batch width is the chunk width, and the activation BO holds exactly that many rows -- an OVER-wide + * answer reads past the BO (engine#1372, memory corruption rather than a slow path). The old code returned the first + * populated design's width, so a mixed set answered whichever role was index 0. + * + * These arms pin the RULE (minimise, skip empty slots, take a fallback), not a measurement: no pack on this box has been + * shown to adopt mixed widths, so nothing here claims the old answer was wrong on a real pack -- only that it would be + * the dangerous way round if one ever did. The sealed route's width is #1904's, read from the plan and measured on the + * device. */ +static void batch_width_arms() { + using onebit::npu::batch_width_min; + const int mixed[] = {16, 16, 128}; + check(batch_width_min(mixed, 3, 1) == 16, "batch width: 16/16/128 answers 16, not the 128 at index 2"); + const int wide_first[] = {128, 16, 16}; + check(batch_width_min(wide_first, 3, 1) == 16, + "batch width: 128 at index 0 does not win (the old code returned the FIRST populated entry)"); + const int slots[] = {0, 0, 16, 0}; + check(batch_width_min(slots, 4, 1) == 16, "batch width: unloaded slots are skipped, not treated as a width"); + const int none[] = {0, 0, 0}; + check(batch_width_min(none, 3, 1) == 1, "batch width: no design at all falls back to 1 (a single-row launch)"); + check(batch_width_min(none, 3, 16) == 16, "batch width: the fallback is the loaded design's own M when the caller has it"); + check(batch_width_min(nullptr, 0, 128) == 128, "batch width: a null array reads as empty and takes the fallback"); + check(batch_width_min(mixed, 0, 4) == 4, "batch width: an n of 0 reads nothing and takes the fallback"); + std::printf(" batch width: 7 arms (min not first, empty slots, fallback)\n"); +} + int main(int argc, char** argv) { + batch_width_arms(); quant_int4_arms(); for (int G : {8, 9, 16}) { for (int n : {G, 3 * G}) { // whole groups: a ragged tail is not a permutation