Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 46 additions & 0 deletions npu/forward/include/batch_width.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
// Copyright 2026 bong-water-water-bong
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// batch_width.h -- how many rows one launch may carry (engine#1900).
//
// The forward chunks a batched pass at this width, and the activation BO holds exactly this many rows: an over-wide
// launch reads and poisons past the BO, which is memory corruption rather than a slow path (engine#1372). So the
// aggregate over a design set must be the MINIMUM of the widths actually loaded, never the first one that happens to
// be populated -- a 16/16/128 set returning 128 corrupts memory on the two narrow roles, while returning 16 is only
// narrower than it needs to be.
//
// Pure and header-only on purpose: the aggregation is the part that can be wrong in the dangerous direction, so it is
// pinned by a host test with no device, no ELF and no runtime.
#pragma once

namespace onebit {
namespace npu {

// The smallest positive width in `widths`, or `fallback` when none is positive.
//
// `widths` may contain 0 for "no design at this role", which is the ordinary state of a design array -- the old code
// returned the FIRST populated entry, so a zero-width slot simply moved the answer to whichever role came next.
inline int batch_width_min(const int* widths, int n, int fallback) {
int w = 0;
if (widths)
for (int i = 0; i < n; ++i) {
const int v = widths[i];
if (v > 0 && (w == 0 || v < w)) w = v;
}
return w > 0 ? w : (fallback > 0 ? fallback : 1);
}

} // namespace npu
} // namespace onebit
4 changes: 4 additions & 0 deletions npu/forward/src/q4nx_forward.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1545,6 +1545,10 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path,

backend_ = "full ELF";
ready_ = true;
/* engine#1900: say the batch width once at load, the way the block note does, so a log can be told apart from a
* forward that never batched. #1904 gives the sealed route its plan's width; this line is what makes the difference
* visible either way. */
std::fprintf(stderr, " [forward] batch width %d row(s) per launch\n", max_batch_rows());
return true;
}

Expand Down
22 changes: 20 additions & 2 deletions npu/target/xdna/xdna_target.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
//
// docs/npu-p4-target-split.md records what moved in A1 and what A2 (plan-driven setup) still owes.
#include "target/xdna/xdna_target.h"
#include "batch_width.h"

#include <algorithm>
#include <chrono>
Expand Down Expand Up @@ -281,8 +282,25 @@ bool XdnaTarget::weight_cached(int layer, int key) const {
}

int XdnaTarget::max_rows() const {
if (pb16i_) { for (int i = 0; i < ndesigns_; i++) if (ctx16_[i]) return ctx16_[i]->MD; }
for (int i = 0; i < ndesigns_; i++) if (ctx_[i]) return ctx_[i]->MD;
/* engine#1900/#1903: MIN over the loaded per-role widths, never the first populated one. This width IS the chunk
* width, and the activation BO holds exactly that many rows -- an over-wide launch reads and poisons past the BO,
* which is memory corruption rather than a slow path (engine#1372). Returning the FIRST entry meant a mixed set
* answered whichever role happened to be index 0, so a 16/16/128 set could answer 128: the dangerous direction.
* The pb16i_ case keeps main's fall-through (a pb16i set that adopted only its prefill kernels has no 16-row twin,
* and answering 1 there would walk single-row passes), and #1904's plan-derived width for the sealed route stays
* below exactly as merged. */
const int n = ndesigns_ < Q4NX_MAX_DESIGNS ? ndesigns_ : Q4NX_MAX_DESIGNS;
int widths16[Q4NX_MAX_DESIGNS] = {0};
bool any16 = false;
if (pb16i_)
for (int i = 0; i < n; i++)
if (ctx16_[i]) { widths16[i] = ctx16_[i]->MD; any16 = true; }
if (any16) return onebit::npu::batch_width_min(widths16, n, 1);
int widths[Q4NX_MAX_DESIGNS] = {0};
bool any_ctx = false;
for (int i = 0; i < n; i++)
if (ctx_[i]) { widths[i] = ctx_[i]->MD; any_ctx = true; }
if (any_ctx) return onebit::npu::batch_width_min(widths, n, 1);
// engine#1900: the sealed/mono-ELF generation adopts its kernel set through a path that fills neither array, so both
// loops above find nothing and this returned 1. This is the CHUNK WIDTH (q4nx_forward's `blk = max_batch_rows()`), so
// those packs batched ONE ROW per launch however wide their design is -- `models/npu/bonsai2-27b-NPU-b8` states prefill
Expand Down
27 changes: 27 additions & 0 deletions tests/npu_pack_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
#include "full_elf.h"
#include "pack.h"
#include "q4nx_pack.h"
#include "batch_width.h"

#include "golden/host_gemm_i8_fixture.h"

Expand Down Expand Up @@ -86,7 +87,33 @@ static void quant_int4_arms() {
std::printf(" int4 pair: 11 arms (known codes, ladder, floor, NaN, round-trip)\n");
}

/* engine#1900: the batch width is the chunk width, and the activation BO holds exactly that many rows -- an OVER-wide
* answer reads past the BO (engine#1372, memory corruption rather than a slow path). The old code returned the first
* populated design's width, so a mixed set answered whichever role was index 0.
*
* These arms pin the RULE (minimise, skip empty slots, take a fallback), not a measurement: no pack on this box has been
* shown to adopt mixed widths, so nothing here claims the old answer was wrong on a real pack -- only that it would be
* the dangerous way round if one ever did. The sealed route's width is #1904's, read from the plan and measured on the
* device. */
static void batch_width_arms() {
using onebit::npu::batch_width_min;
const int mixed[] = {16, 16, 128};
check(batch_width_min(mixed, 3, 1) == 16, "batch width: 16/16/128 answers 16, not the 128 at index 2");
const int wide_first[] = {128, 16, 16};
check(batch_width_min(wide_first, 3, 1) == 16,
"batch width: 128 at index 0 does not win (the old code returned the FIRST populated entry)");
const int slots[] = {0, 0, 16, 0};
check(batch_width_min(slots, 4, 1) == 16, "batch width: unloaded slots are skipped, not treated as a width");
const int none[] = {0, 0, 0};
check(batch_width_min(none, 3, 1) == 1, "batch width: no design at all falls back to 1 (a single-row launch)");
check(batch_width_min(none, 3, 16) == 16, "batch width: the fallback is the loaded design's own M when the caller has it");
check(batch_width_min(nullptr, 0, 128) == 128, "batch width: a null array reads as empty and takes the fallback");
check(batch_width_min(mixed, 0, 4) == 4, "batch width: an n of 0 reads nothing and takes the fallback");
std::printf(" batch width: 7 arms (min not first, empty slots, fallback)\n");
}

int main(int argc, char** argv) {
batch_width_arms();
quant_int4_arms();
for (int G : {8, 9, 16}) {
for (int n : {G, 3 * G}) { // whole groups: a ragged tail is not a permutation
Expand Down
Loading