diff --git a/npu/forward/include/batch_width.h b/npu/forward/include/batch_width.h new file mode 100644 index 000000000..9901849a9 --- /dev/null +++ b/npu/forward/include/batch_width.h @@ -0,0 +1,46 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// batch_width.h -- how many rows one launch may carry (engine#1900). +// +// The forward chunks a batched pass at this width, and the activation BO holds exactly this many rows: an over-wide +// launch reads and poisons past the BO, which is memory corruption rather than a slow path (engine#1372). So the +// aggregate over a design set must be the MINIMUM of the widths actually loaded, never the first one that happens to +// be populated -- a 16/16/128 set returning 128 corrupts memory on the two narrow roles, while returning 16 is only +// narrower than it needs to be. +// +// Pure and header-only on purpose: the aggregation is the part that can be wrong in the dangerous direction, so it is +// pinned by a host test with no device, no ELF and no runtime. +#pragma once + +namespace onebit { +namespace npu { + +// The smallest positive width in `widths`, or `fallback` when none is positive. +// +// `widths` may contain 0 for "no design at this role", which is the ordinary state of a design array -- the old code +// returned the FIRST populated entry, so a zero-width slot simply moved the answer to whichever role came next. +inline int batch_width_min(const int* widths, int n, int fallback) { + int w = 0; + if (widths) + for (int i = 0; i < n; ++i) { + const int v = widths[i]; + if (v > 0 && (w == 0 || v < w)) w = v; + } + return w > 0 ? w : (fallback > 0 ? fallback : 1); +} + +} // namespace npu +} // namespace onebit diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index b587a1eb6..172af14fd 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -1545,6 +1545,13 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, backend_ = "full ELF"; ready_ = true; + /* engine#1900: say the batch width once at load, the way the block note does, so a log can be told apart from a + * forward that never batched. Before this, a sealed pack reported 1 through the whole run and nothing said so -- + * the only symptom was N single-row passes and #1781 gate 1 having no pack to measure on. Read once: the value + * printed is the value tested. */ + const int batch_w = max_batch_rows(); + std::fprintf(stderr, " [forward] batch width %d row(s) per launch%s\n", batch_w, + batch_w <= 1 ? " (single row: no per-role design width and the plan states no prefill M)" : ""); return true; } diff --git a/npu/target/xdna/xdna_target.cpp b/npu/target/xdna/xdna_target.cpp index 5a3621a93..b44bbf3e9 100644 --- a/npu/target/xdna/xdna_target.cpp +++ b/npu/target/xdna/xdna_target.cpp @@ -20,6 +20,7 @@ // // docs/npu-p4-target-split.md records what moved in A1 and what A2 (plan-driven setup) still owes. #include "target/xdna/xdna_target.h" +#include "batch_width.h" #include #include @@ -281,17 +282,31 @@ bool XdnaTarget::weight_cached(int layer, int key) const { } int XdnaTarget::max_rows() const { - if (pb16i_) { for (int i = 0; i < ndesigns_; i++) if (ctx16_[i]) return ctx16_[i]->MD; } - for (int i = 0; i < ndesigns_; i++) if (ctx_[i]) return ctx_[i]->MD; - // engine#1900: the sealed/mono-ELF generation adopts its kernel set through a path that fills neither array, so both - // loops above find nothing and this returned 1. This is the CHUNK WIDTH (q4nx_forward's `blk = max_batch_rows()`), so - // those packs batched ONE ROW per launch however wide their design is -- `models/npu/bonsai2-27b-NPU-b8` states prefill - // M=128 and reported 1. Only reachable when no per-role context was adopted, so nothing that has one changes; and the - // plan is the pack's own statement of the width its kernels were built for (a plan whose M disagrees with adopted - // kernels is already refused above). Still 1 when the plan states nothing: guessing a width launches into a BO sized for - // fewer rows, which is the memory-corruption path engine#1372 records. - if (plan_.prefill_m > 0) return plan_.prefill_m; - return 1; + /* engine#1900: this width IS the chunk width, and the activation BO holds exactly that many rows -- an over-wide + * launch reads and poisons past the BO, which is memory corruption rather than a slow path (engine#1372). The old + * code returned the FIRST populated per-role entry, so a mixed set answered whichever role happened to be index 0. + * + * The MINIMUM of the loaded widths is the only safe aggregate, and the pb16i_ case keeps main's FALL-THROUGH: a + * pb16i set that adopted only its prefill kernels has no 16-row twin, and answering 1 there would walk single-row + * passes (the #1900 symptom on a different route). + * + * The sealed/mono-ELF generation adopts its kernel set through a path that fills neither array, so both aggregates + * find nothing. Its answer is the plan's prefill M (engine#1904, measured on the device: bonsai2-27b-NPU-b8 states + * M=128 and reported 1): the plan is the pack's own statement of the width its kernels were built for, and a plan + * whose M disagrees with adopted kernels is already refused at load. It is only the FALLBACK -- a loaded per-role + * design always wins, so this never answers wider than the narrowest loaded design -- and it stays 1 when the plan + * states nothing: guessing a width launches into a BO sized for fewer rows. */ + const int n = ndesigns_ < Q4NX_MAX_DESIGNS ? ndesigns_ : Q4NX_MAX_DESIGNS; + int widths16[Q4NX_MAX_DESIGNS] = {0}; + bool any16 = false; + if (pb16i_) + for (int i = 0; i < n; i++) + if (ctx16_[i]) { widths16[i] = ctx16_[i]->MD; any16 = true; } + if (any16) return onebit::npu::batch_width_min(widths16, n, 1); + int widths[Q4NX_MAX_DESIGNS] = {0}; + for (int i = 0; i < n; i++) + if (ctx_[i]) widths[i] = ctx_[i]->MD; + return onebit::npu::batch_width_min(widths, n, plan_.prefill_m); } int XdnaTarget::rows_for(int role, int want) const { diff --git a/tests/npu_pack_test.cpp b/tests/npu_pack_test.cpp index a544d9601..fa652c794 100644 --- a/tests/npu_pack_test.cpp +++ b/tests/npu_pack_test.cpp @@ -23,6 +23,7 @@ #include "full_elf.h" #include "pack.h" #include "q4nx_pack.h" +#include "batch_width.h" #include "golden/host_gemm_i8_fixture.h" @@ -86,7 +87,42 @@ static void quant_int4_arms() { std::printf(" int4 pair: 11 arms (known codes, ladder, floor, NaN, round-trip)\n"); } +/* engine#1900: the batch width is the chunk width, and the activation BO holds exactly that many rows -- an OVER-wide + * answer reads past the BO (engine#1372, memory corruption rather than a slow path). The old code returned the first + * populated design's width, so a mixed set answered whichever role was index 0. + * + * These arms pin the RULE (minimise, skip empty slots, take a fallback), not a measurement: no pack on this box has been + * shown to adopt mixed widths, so nothing here claims the old answer was wrong on a real pack -- only that it would be + * the dangerous way round if one ever did. The sealed route's width is #1904's, read from the plan and measured on the + * device. */ +static void batch_width_arms() { + using onebit::npu::batch_width_min; + const int mixed[] = {16, 16, 128}; + check(batch_width_min(mixed, 3, 1) == 16, "batch width: 16/16/128 answers 16, not the 128 at index 2"); + const int wide_first[] = {128, 16, 16}; + check(batch_width_min(wide_first, 3, 1) == 16, + "batch width: 128 at index 0 does not win (the old code returned the FIRST populated entry)"); + const int slots[] = {0, 0, 16, 0}; + check(batch_width_min(slots, 4, 1) == 16, "batch width: unloaded slots are skipped, not treated as a width"); + const int none[] = {0, 0, 0}; + check(batch_width_min(none, 3, 1) == 1, "batch width: no design at all falls back to 1 (a single-row launch)"); + check(batch_width_min(none, 3, 16) == 16, "batch width: the fallback is the loaded design's own M when the caller has it"); + check(batch_width_min(nullptr, 0, 128) == 128, "batch width: a null array reads as empty and takes the fallback"); + check(batch_width_min(mixed, 0, 4) == 4, "batch width: an n of 0 reads nothing and takes the fallback"); + /* The sealed route (engine#1904): no per-role design is adopted, so both arrays are empty and the answer is the + * plan's prefill M, passed as the fallback. These are the arms that would have caught a merge that dropped it. */ + const int plan_m = 128; + check(batch_width_min(none, 3, plan_m) == 128, + "batch width: sealed route (no per-role design) answers the plan's prefill M, not 1"); + check(batch_width_min(none, 3, 0) == 1, "batch width: sealed route with a plan that states no M stays at 1"); + const int narrow[] = {0, 16, 0}; + check(batch_width_min(narrow, 3, plan_m) == 16, + "batch width: a loaded 16-row design beats a wider plan M (the plan is only the fallback)"); + std::printf(" batch width: 10 arms (min not first, empty slots, fallback, sealed plan M)\n"); +} + int main(int argc, char** argv) { + batch_width_arms(); quant_int4_arms(); for (int G : {8, 9, 16}) { for (int n : {G, 3 * G}) { // whole groups: a ragged tail is not a permutation