Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 46 additions & 0 deletions npu/forward/include/batch_width.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
// Copyright 2026 bong-water-water-bong
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// batch_width.h -- how many rows one launch may carry (engine#1900).
//
// The forward chunks a batched pass at this width, and the activation BO holds exactly this many rows: an over-wide
// launch reads and poisons past the BO, which is memory corruption rather than a slow path (engine#1372). So the
// aggregate over a design set must be the MINIMUM of the widths actually loaded, never the first one that happens to
// be populated -- a 16/16/128 set returning 128 corrupts memory on the two narrow roles, while returning 16 is only
// narrower than it needs to be.
//
// Pure and header-only on purpose: the aggregation is the part that can be wrong in the dangerous direction, so it is
// pinned by a host test with no device, no ELF and no runtime.
#pragma once

namespace onebit {
namespace npu {

// The smallest positive width in `widths`, or `fallback` when none is positive.
//
// `widths` may contain 0 for "no design at this role", which is the ordinary state of a design array -- the old code
// returned the FIRST populated entry, so a zero-width slot simply moved the answer to whichever role came next.
inline int batch_width_min(const int* widths, int n, int fallback) {
int w = 0;
if (widths)
for (int i = 0; i < n; ++i) {
const int v = widths[i];
if (v > 0 && (w == 0 || v < w)) w = v;
}
return w > 0 ? w : (fallback > 0 ? fallback : 1);
}

} // namespace npu
} // namespace onebit
7 changes: 7 additions & 0 deletions npu/forward/src/q4nx_forward.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1545,6 +1545,13 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path,

backend_ = "full ELF";
ready_ = true;
/* engine#1900: say the batch width once at load, the way the block note does, so a log can be told apart from a
* forward that never batched. Before this, a sealed pack reported 1 through the whole run and nothing said so --
* the only symptom was N single-row passes and #1781 gate 1 having no pack to measure on. Read once: the value
* printed is the value tested. */
const int batch_w = max_batch_rows();
std::fprintf(stderr, " [forward] batch width %d row(s) per launch%s\n", batch_w,
batch_w <= 1 ? " (single row: no per-role design width and the plan states no prefill M)" : "");
return true;
}

Expand Down
37 changes: 26 additions & 11 deletions npu/target/xdna/xdna_target.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
//
// docs/npu-p4-target-split.md records what moved in A1 and what A2 (plan-driven setup) still owes.
#include "target/xdna/xdna_target.h"
#include "batch_width.h"

#include <algorithm>
#include <chrono>
Expand Down Expand Up @@ -281,17 +282,31 @@ bool XdnaTarget::weight_cached(int layer, int key) const {
}

int XdnaTarget::max_rows() const {
if (pb16i_) { for (int i = 0; i < ndesigns_; i++) if (ctx16_[i]) return ctx16_[i]->MD; }
for (int i = 0; i < ndesigns_; i++) if (ctx_[i]) return ctx_[i]->MD;
// engine#1900: the sealed/mono-ELF generation adopts its kernel set through a path that fills neither array, so both
// loops above find nothing and this returned 1. This is the CHUNK WIDTH (q4nx_forward's `blk = max_batch_rows()`), so
// those packs batched ONE ROW per launch however wide their design is -- `models/npu/bonsai2-27b-NPU-b8` states prefill
// M=128 and reported 1. Only reachable when no per-role context was adopted, so nothing that has one changes; and the
// plan is the pack's own statement of the width its kernels were built for (a plan whose M disagrees with adopted
// kernels is already refused above). Still 1 when the plan states nothing: guessing a width launches into a BO sized for
// fewer rows, which is the memory-corruption path engine#1372 records.
if (plan_.prefill_m > 0) return plan_.prefill_m;
return 1;
/* engine#1900: this width IS the chunk width, and the activation BO holds exactly that many rows -- an over-wide
* launch reads and poisons past the BO, which is memory corruption rather than a slow path (engine#1372). The old
* code returned the FIRST populated per-role entry, so a mixed set answered whichever role happened to be index 0.
*
* The MINIMUM of the loaded widths is the only safe aggregate, and the pb16i_ case keeps main's FALL-THROUGH: a
* pb16i set that adopted only its prefill kernels has no 16-row twin, and answering 1 there would walk single-row
* passes (the #1900 symptom on a different route).
*
* The sealed/mono-ELF generation adopts its kernel set through a path that fills neither array, so both aggregates
* find nothing. Its answer is the plan's prefill M (engine#1904, measured on the device: bonsai2-27b-NPU-b8 states
* M=128 and reported 1): the plan is the pack's own statement of the width its kernels were built for, and a plan
* whose M disagrees with adopted kernels is already refused at load. It is only the FALLBACK -- a loaded per-role
* design always wins, so this never answers wider than the narrowest loaded design -- and it stays 1 when the plan
* states nothing: guessing a width launches into a BO sized for fewer rows. */
const int n = ndesigns_ < Q4NX_MAX_DESIGNS ? ndesigns_ : Q4NX_MAX_DESIGNS;
int widths16[Q4NX_MAX_DESIGNS] = {0};
bool any16 = false;
if (pb16i_)
for (int i = 0; i < n; i++)
if (ctx16_[i]) { widths16[i] = ctx16_[i]->MD; any16 = true; }
if (any16) return onebit::npu::batch_width_min(widths16, n, 1);
int widths[Q4NX_MAX_DESIGNS] = {0};
for (int i = 0; i < n; i++)
if (ctx_[i]) widths[i] = ctx_[i]->MD;
return onebit::npu::batch_width_min(widths, n, plan_.prefill_m);
}

int XdnaTarget::rows_for(int role, int want) const {
Expand Down
36 changes: 36 additions & 0 deletions tests/npu_pack_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
#include "full_elf.h"
#include "pack.h"
#include "q4nx_pack.h"
#include "batch_width.h"

#include "golden/host_gemm_i8_fixture.h"

Expand Down Expand Up @@ -86,7 +87,42 @@ static void quant_int4_arms() {
std::printf(" int4 pair: 11 arms (known codes, ladder, floor, NaN, round-trip)\n");
}

/* engine#1900: the batch width is the chunk width, and the activation BO holds exactly that many rows -- an OVER-wide
* answer reads past the BO (engine#1372, memory corruption rather than a slow path). The old code returned the first
* populated design's width, so a mixed set answered whichever role was index 0.
*
* These arms pin the RULE (minimise, skip empty slots, take a fallback), not a measurement: no pack on this box has been
* shown to adopt mixed widths, so nothing here claims the old answer was wrong on a real pack -- only that it would be
* the dangerous way round if one ever did. The sealed route's width is #1904's, read from the plan and measured on the
* device. */
static void batch_width_arms() {
using onebit::npu::batch_width_min;
const int mixed[] = {16, 16, 128};
check(batch_width_min(mixed, 3, 1) == 16, "batch width: 16/16/128 answers 16, not the 128 at index 2");
const int wide_first[] = {128, 16, 16};
check(batch_width_min(wide_first, 3, 1) == 16,
"batch width: 128 at index 0 does not win (the old code returned the FIRST populated entry)");
const int slots[] = {0, 0, 16, 0};
check(batch_width_min(slots, 4, 1) == 16, "batch width: unloaded slots are skipped, not treated as a width");
const int none[] = {0, 0, 0};
check(batch_width_min(none, 3, 1) == 1, "batch width: no design at all falls back to 1 (a single-row launch)");
check(batch_width_min(none, 3, 16) == 16, "batch width: the fallback is the loaded design's own M when the caller has it");
check(batch_width_min(nullptr, 0, 128) == 128, "batch width: a null array reads as empty and takes the fallback");
check(batch_width_min(mixed, 0, 4) == 4, "batch width: an n of 0 reads nothing and takes the fallback");
/* The sealed route (engine#1904): no per-role design is adopted, so both arrays are empty and the answer is the
* plan's prefill M, passed as the fallback. These are the arms that would have caught a merge that dropped it. */
const int plan_m = 128;
check(batch_width_min(none, 3, plan_m) == 128,
"batch width: sealed route (no per-role design) answers the plan's prefill M, not 1");
check(batch_width_min(none, 3, 0) == 1, "batch width: sealed route with a plan that states no M stays at 1");
const int narrow[] = {0, 16, 0};
check(batch_width_min(narrow, 3, plan_m) == 16,
"batch width: a loaded 16-row design beats a wider plan M (the plan is only the fallback)");
std::printf(" batch width: 10 arms (min not first, empty slots, fallback, sealed plan M)\n");
}

int main(int argc, char** argv) {
batch_width_arms();
quant_int4_arms();
for (int G : {8, 9, 16}) {
for (int n : {G, 3 * G}) { // whole groups: a ragged tail is not a permutation
Expand Down
Loading