Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -857,6 +857,17 @@ add_executable(npu_elf_args_test tests/npu_elf_args_test.c npu/forward/src/q4nx_
target_include_directories(npu_elf_args_test PRIVATE npu/forward/include)
target_link_libraries(npu_elf_args_test PRIVATE m)
add_test(NAME npu_elf_args COMMAND npu_elf_args_test)
# The batched verify pass's chunk arithmetic (engine#1386/#1364): no launch wider than the context's row
# capacity, contiguous coverage of every row, and no chunks at all when the capacity is unknown. Arithmetic
# only, so it runs everywhere, including CI without a device.
add_executable(npu_row_chunks_test tests/npu_row_chunks_test.cpp)
target_include_directories(npu_row_chunks_test PRIVATE ${CMAKE_SOURCE_DIR}/npu/forward/include)
add_test(NAME npu_row_chunks COMMAND npu_row_chunks_test)
# ci.yml builds a named list of targets (its own comment: "keep this list in step with add_test()"), and the
# PR token cannot edit a workflow file, so the test binary is built by way of a target that IS in that list -
# the same arrangement npu_plan_test already has below. Without this, `ctest` reports the test "Not Run":
# it only runs binaries that exist.
add_dependencies(npu_elf_args_test npu_row_chunks_test)
add_executable(npu_plan_test tests/npu_plan_test.cpp)
target_include_directories(npu_plan_test PRIVATE ${CMAKE_SOURCE_DIR}/npu)
target_link_libraries(npu_plan_test PRIVATE nlohmann_json::nlohmann_json)
Expand Down
62 changes: 62 additions & 0 deletions npu/forward/include/q4nx_row_chunks.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
// Copyright 2026 bong-water-water-bong
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// q4nx_row_chunks.h -- how a batched verify pass is split into device launches.
//
// A hardware context has a fixed row capacity (I8Ctx::MD: 16 for the decode twins, 128 for the prefill
// design) and `quantize_async_rows()` REFUSES a wider batch -- it prints "am=... exceeds bA capacity"
// and returns the rows unquantised, after which the caller's launch still runs. That is not a slow
// path, it is a read and a write past the end of the activation BO: engine#1372 was exactly that on the
// per-op prefill path, and engine#1386 is the same shape on the pb16i LM head, whose rows path launched
// ALL of M into a 16-row context.
//
// #1364 fixed the int8 head by launching in chunks of at most MD rows (a resident weight tile, one
// launch per chunk, and a row's activation scale is its own either way, so chunking cannot move a
// value). This header is that arithmetic, in one place, with the invariant the tests pin:
//
// * no chunk is wider than MD;
// * the chunks are contiguous and cover [0, M) exactly once, in order;
// * M == 0 is no chunks at all;
// * an unknown capacity (MD <= 0) is no chunks, so the caller falls back to its per-row path instead
// of launching blind.
//
// NPU_HEAD_NO_CHUNK=1 asks for the pre-chunking arm: one chunk per row. That is the A/B arm for the
// claim (#1364's knob, reused so both heads answer the same way), never the default.
#pragma once

#include <algorithm>
#include <vector>

namespace onebit {
namespace npu {

struct RowChunk {
int offset; // first row of the chunk, in [0, M)
int rows; // 1 .. MD
};

// The chunks a batched launch of `rows` rows is made in, given a context capacity of `capacity` rows.
// `chunked == false` gives one chunk per row (the NPU_HEAD_NO_CHUNK arm).
inline std::vector<RowChunk> row_chunks(int rows, int capacity, bool chunked = true) {
std::vector<RowChunk> out;
if (rows <= 0) return out;
if (capacity <= 0) return out; // unknown capacity: the caller must not launch
const int step = chunked ? std::min(rows, capacity) : 1;
for (int r0 = 0; r0 < rows; r0 += step) out.push_back(RowChunk{r0, std::min(step, rows - r0)});
return out;
}

} // namespace npu
} // namespace onebit
52 changes: 38 additions & 14 deletions npu/forward/src/q4nx_forward.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
// Numerics follow tools/q4nx_forward_ref.py so the two can be compared
// element-wise.
#include "q4nx_forward.h"
#include "q4nx_row_chunks.h"
#include "q4nx_state_planes.h" // the plane-copy half of a snapshot, device-free and unit-tested
#include <algorithm>
#include <fcntl.h>
Expand Down Expand Up @@ -4898,21 +4899,44 @@ bool Q4nxNpuForward::run_lm_head_rows(const std::vector<float>& h_rows, int M,
scales[(size_t)m] = amax > 0.0f ? amax / 127.0f : 1.0f;
}
I8Ctx& lctx = *xd_->head_ctx();
std::vector<float> got((size_t)M * (size_t)T);
for (int t = 0; t < ntiles; t++) {
const int n0 = t * T, n1 = std::min(NV, n0 + T);
xrt::run run = lctx.launch_async_with_bo_rows(*lm_head_packed_[(size_t)t].bo, hr.data(), M, H,
scales.data());
lctx.wait_kernel(run);
lctx.readback();
std::fill(got.begin(), got.end(), 0.0f);
lctx.dequant_pb16i(got.data(), M, T, scales.data(), lm_head_packed_[(size_t)t].eref,
xd_->head_ctx()->pb16i_planes);
// engine#1386: the head context is built with MD = Q4NX_DESIGN_M_DECODE (16) rows, and this path used
// to launch ALL M rows into it. quantize_async_rows() refuses a wider batch and returns the rows
// unquantised, and `launch_async_with_bo_rows` then ran anyway -- a read and a write past the end of
// the activation BO, which is #1372's failure on the head. Chunk at the context's own capacity, the
// way #1364 did for the int8 head: the weights are resident, so a chunk is one launch, and each row's
// rotation and scale are its own, so chunking cannot move a value. NPU_HEAD_NO_CHUNK=1 is the
// pre-chunking arm (one row per chunk) that makes an A/B possible on the same artifacts.
static const bool chunk_on = std::getenv("NPU_HEAD_NO_CHUNK") == nullptr;
const std::vector<onebit::npu::RowChunk> chunks = onebit::npu::row_chunks(M, lctx.MD, chunk_on);
if (chunks.empty() && M > 0) {
// No capacity to launch into: the per-row path below is the correct answer, not a blind launch.
for (int m = 0; m < M; m++) {
for (int n = n0; n < n1; n++) {
float v = got[(size_t)m * T + (n - n0)];
if (sem_.logit_softcap > 0) v = sem_.logit_softcap * std::tanh(v / sem_.logit_softcap);
logits_all[(size_t)m * (size_t)NV + (size_t)n] = v;
std::vector<float> one(h_rows.begin() + (size_t)m * H, h_rows.begin() + (size_t)(m + 1) * H);
std::vector<float> lg;
if (!run_lm_head(one, lg)) return false;
std::copy(lg.begin(), lg.end(), logits_all.begin() + (size_t)m * (size_t)NV);
}
return true;
}
for (const onebit::npu::RowChunk& c : chunks) {
const int r0 = c.offset, rows = c.rows;
std::vector<float> got((size_t)rows * (size_t)T);
for (int t = 0; t < ntiles; t++) {
const int n0 = t * T, n1 = std::min(NV, n0 + T);
xrt::run run = lctx.launch_async_with_bo_rows(
*lm_head_packed_[(size_t)t].bo, hr.data() + (size_t)r0 * (size_t)H, rows, H,
scales.data() + r0);
lctx.wait_kernel(run);
lctx.readback();
std::fill(got.begin(), got.end(), 0.0f);
lctx.dequant_pb16i(got.data(), rows, T, scales.data() + r0, lm_head_packed_[(size_t)t].eref,
xd_->head_ctx()->pb16i_planes);
for (int m = 0; m < rows; m++) {
for (int n = n0; n < n1; n++) {
float v = got[(size_t)m * T + (n - n0)];
if (sem_.logit_softcap > 0) v = sem_.logit_softcap * std::tanh(v / sem_.logit_softcap);
logits_all[(size_t)(r0 + m) * (size_t)NV + (size_t)n] = v;
}
}
}
}
Expand Down
113 changes: 113 additions & 0 deletions tests/npu_row_chunks_test.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
// Copyright 2026 bong-water-water-bong
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// npu_row_chunks_test.cpp -- the invariant that keeps a batched verify pass inside its context.
//
// engine#1386: the pb16i LM-head rows path launched all M rows into a 16-row context; the quantiser
// refused the batch, returned the rows unquantised, and the launch ran anyway (engine#1372's failure on
// the head). The fix is the chunking #1364 shipped for the int8 head, and these are the properties that
// make it a fix rather than a rearrangement:
//
// no chunk wider than the capacity, the chunks contiguous and in order, every row covered exactly
// once, M == 0 as no chunks, an unknown capacity as no chunks (so the caller falls back instead of
// launching blind), and the A/B arm (one row per chunk) as the same coverage.
//
// Device-free: the header is arithmetic only. A device A/B of the launch count is #1386's own note.
#include <cstdio>
#include <set>
#include <vector>

#include "q4nx_row_chunks.h"

namespace {

int g_failures = 0;

void check(bool cond, const char* what) {
if (!cond) {
std::printf("FAIL %s\n", what);
++g_failures;
}
}

// Coverage: contiguous from 0, no gaps, no overlaps, total == rows, each chunk <= capacity, all >= 1.
void check_cover(int rows, int capacity) {
const std::vector<onebit::npu::RowChunk> cs = onebit::npu::row_chunks(rows, capacity, true);
int next = 0;
std::set<int> seen;
for (const auto& c : cs) {
check(c.rows >= 1, "chunk is non-empty");
check(c.rows <= capacity, "chunk is no wider than the capacity");
check(c.offset == next, "chunk is contiguous with the previous one");
for (int r = c.offset; r < c.offset + c.rows; ++r) check(seen.insert(r).second, "row covered once");
next = c.offset + c.rows;
}
check(next == rows, "chunks cover every row");
check((int)cs.size() == (rows + capacity - 1) / capacity, "chunk count is ceil(rows / capacity)");
}

} // namespace

int main() {
using onebit::npu::row_chunks;

// 1. The regression itself: M above the 16-row twin capacity must become ceil(M/16) chunks, never one
// launch of M rows. The old code passed M straight through, which is what the device refused.
for (int m : {17, 18, 31, 32, 33, 64, 128}) check_cover(m, 16);
check((int)row_chunks(17, 16, true).size() == 2, "17 rows into a 16-row context is 2 launches");
check((int)row_chunks(128, 16, true).size() == 8, "a 128-row verify batch is 8 launches");
check(row_chunks(17, 16, true)[0].offset == 0 && row_chunks(17, 16, true)[1].offset == 16,
"the second chunk starts at row 16");
check(row_chunks(17, 16, true)[1].rows == 1, "the tail chunk carries the 1 leftover row");

// 2. Small and exact batches are unchanged: M <= capacity is one launch of M rows (so M=16 is
// bit-for-bit what the path did before chunking).
for (int m : {1, 2, 16}) {
const auto cs = row_chunks(m, 16, true);
check((int)cs.size() == 1 && cs[0].offset == 0 && cs[0].rows == m, "M <= capacity is one launch");
}
check((int)row_chunks(16, 16, true).size() == 1, "M == capacity is exactly one launch");
check_cover(1, 16);
check_cover(16, 16);

// 3. The prefill design's capacity (128 rows) is where the head would never need chunking, but the
// arithmetic must still hold for it.
check((int)row_chunks(128, 128, true).size() == 1, "128 rows into a 128-row context is one launch");
check_cover(130, 128);

// 4. The A/B arm (NPU_HEAD_NO_CHUNK=1): one row per chunk, same coverage - that is what the path did
// before chunking existed, so a numeric A/B compares artifacts, not coverage.
const auto per_row = row_chunks(17, 16, false);
check((int)per_row.size() == 17, "the no-chunk arm is one launch per row");
int next = 0;
for (const auto& c : per_row) {
check(c.rows == 1 && c.offset == next, "no-chunk arm covers rows one at a time, in order");
++next;
}

// 5. Refusals, not guesses: no rows and an unknown capacity both produce no chunks, so the caller
// takes its per-row path rather than launching into a context it cannot size.
check(row_chunks(0, 16, true).empty(), "0 rows is no chunks");
check(row_chunks(-3, 16, true).empty(), "a negative row count is no chunks");
check(row_chunks(64, 0, true).empty(), "capacity 0 is no chunks");
check(row_chunks(64, -1, true).empty(), "a negative capacity is no chunks");

if (g_failures == 0) {
std::printf("npu_row_chunks: PASS\n");
return 0;
}
std::printf("npu_row_chunks: %d failure(s)\n", g_failures);
return 1;
}
Loading