diff --git a/CMakeLists.txt b/CMakeLists.txt index b52159a21..2b00699aa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -857,6 +857,17 @@ add_executable(npu_elf_args_test tests/npu_elf_args_test.c npu/forward/src/q4nx_ target_include_directories(npu_elf_args_test PRIVATE npu/forward/include) target_link_libraries(npu_elf_args_test PRIVATE m) add_test(NAME npu_elf_args COMMAND npu_elf_args_test) +# The batched verify pass's chunk arithmetic (engine#1386/#1364): no launch wider than the context's row +# capacity, contiguous coverage of every row, and no chunks at all when the capacity is unknown. Arithmetic +# only, so it runs everywhere, including CI without a device. +add_executable(npu_row_chunks_test tests/npu_row_chunks_test.cpp) +target_include_directories(npu_row_chunks_test PRIVATE ${CMAKE_SOURCE_DIR}/npu/forward/include) +add_test(NAME npu_row_chunks COMMAND npu_row_chunks_test) +# ci.yml builds a named list of targets (its own comment: "keep this list in step with add_test()"), and the +# PR token cannot edit a workflow file, so the test binary is built by way of a target that IS in that list - +# the same arrangement npu_plan_test already has below. Without this, `ctest` reports the test "Not Run": +# it only runs binaries that exist. +add_dependencies(npu_elf_args_test npu_row_chunks_test) add_executable(npu_plan_test tests/npu_plan_test.cpp) target_include_directories(npu_plan_test PRIVATE ${CMAKE_SOURCE_DIR}/npu) target_link_libraries(npu_plan_test PRIVATE nlohmann_json::nlohmann_json) diff --git a/npu/forward/include/q4nx_row_chunks.h b/npu/forward/include/q4nx_row_chunks.h new file mode 100644 index 000000000..93b4b4521 --- /dev/null +++ b/npu/forward/include/q4nx_row_chunks.h @@ -0,0 +1,62 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// q4nx_row_chunks.h -- how a batched verify pass is split into device launches. +// +// A hardware context has a fixed row capacity (I8Ctx::MD: 16 for the decode twins, 128 for the prefill +// design) and `quantize_async_rows()` REFUSES a wider batch -- it prints "am=... exceeds bA capacity" +// and returns the rows unquantised, after which the caller's launch still runs. That is not a slow +// path, it is a read and a write past the end of the activation BO: engine#1372 was exactly that on the +// per-op prefill path, and engine#1386 is the same shape on the pb16i LM head, whose rows path launched +// ALL of M into a 16-row context. +// +// #1364 fixed the int8 head by launching in chunks of at most MD rows (a resident weight tile, one +// launch per chunk, and a row's activation scale is its own either way, so chunking cannot move a +// value). This header is that arithmetic, in one place, with the invariant the tests pin: +// +// * no chunk is wider than MD; +// * the chunks are contiguous and cover [0, M) exactly once, in order; +// * M == 0 is no chunks at all; +// * an unknown capacity (MD <= 0) is no chunks, so the caller falls back to its per-row path instead +// of launching blind. +// +// NPU_HEAD_NO_CHUNK=1 asks for the pre-chunking arm: one chunk per row. That is the A/B arm for the +// claim (#1364's knob, reused so both heads answer the same way), never the default. +#pragma once + +#include +#include + +namespace onebit { +namespace npu { + +struct RowChunk { + int offset; // first row of the chunk, in [0, M) + int rows; // 1 .. MD +}; + +// The chunks a batched launch of `rows` rows is made in, given a context capacity of `capacity` rows. +// `chunked == false` gives one chunk per row (the NPU_HEAD_NO_CHUNK arm). +inline std::vector row_chunks(int rows, int capacity, bool chunked = true) { + std::vector out; + if (rows <= 0) return out; + if (capacity <= 0) return out; // unknown capacity: the caller must not launch + const int step = chunked ? std::min(rows, capacity) : 1; + for (int r0 = 0; r0 < rows; r0 += step) out.push_back(RowChunk{r0, std::min(step, rows - r0)}); + return out; +} + +} // namespace npu +} // namespace onebit diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index ea3dd188b..30ed73ae8 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -24,6 +24,7 @@ // Numerics follow tools/q4nx_forward_ref.py so the two can be compared // element-wise. #include "q4nx_forward.h" +#include "q4nx_row_chunks.h" #include "q4nx_state_planes.h" // the plane-copy half of a snapshot, device-free and unit-tested #include #include @@ -4898,21 +4899,44 @@ bool Q4nxNpuForward::run_lm_head_rows(const std::vector& h_rows, int M, scales[(size_t)m] = amax > 0.0f ? amax / 127.0f : 1.0f; } I8Ctx& lctx = *xd_->head_ctx(); - std::vector got((size_t)M * (size_t)T); - for (int t = 0; t < ntiles; t++) { - const int n0 = t * T, n1 = std::min(NV, n0 + T); - xrt::run run = lctx.launch_async_with_bo_rows(*lm_head_packed_[(size_t)t].bo, hr.data(), M, H, - scales.data()); - lctx.wait_kernel(run); - lctx.readback(); - std::fill(got.begin(), got.end(), 0.0f); - lctx.dequant_pb16i(got.data(), M, T, scales.data(), lm_head_packed_[(size_t)t].eref, - xd_->head_ctx()->pb16i_planes); + // engine#1386: the head context is built with MD = Q4NX_DESIGN_M_DECODE (16) rows, and this path used + // to launch ALL M rows into it. quantize_async_rows() refuses a wider batch and returns the rows + // unquantised, and `launch_async_with_bo_rows` then ran anyway -- a read and a write past the end of + // the activation BO, which is #1372's failure on the head. Chunk at the context's own capacity, the + // way #1364 did for the int8 head: the weights are resident, so a chunk is one launch, and each row's + // rotation and scale are its own, so chunking cannot move a value. NPU_HEAD_NO_CHUNK=1 is the + // pre-chunking arm (one row per chunk) that makes an A/B possible on the same artifacts. + static const bool chunk_on = std::getenv("NPU_HEAD_NO_CHUNK") == nullptr; + const std::vector chunks = onebit::npu::row_chunks(M, lctx.MD, chunk_on); + if (chunks.empty() && M > 0) { + // No capacity to launch into: the per-row path below is the correct answer, not a blind launch. for (int m = 0; m < M; m++) { - for (int n = n0; n < n1; n++) { - float v = got[(size_t)m * T + (n - n0)]; - if (sem_.logit_softcap > 0) v = sem_.logit_softcap * std::tanh(v / sem_.logit_softcap); - logits_all[(size_t)m * (size_t)NV + (size_t)n] = v; + std::vector one(h_rows.begin() + (size_t)m * H, h_rows.begin() + (size_t)(m + 1) * H); + std::vector lg; + if (!run_lm_head(one, lg)) return false; + std::copy(lg.begin(), lg.end(), logits_all.begin() + (size_t)m * (size_t)NV); + } + return true; + } + for (const onebit::npu::RowChunk& c : chunks) { + const int r0 = c.offset, rows = c.rows; + std::vector got((size_t)rows * (size_t)T); + for (int t = 0; t < ntiles; t++) { + const int n0 = t * T, n1 = std::min(NV, n0 + T); + xrt::run run = lctx.launch_async_with_bo_rows( + *lm_head_packed_[(size_t)t].bo, hr.data() + (size_t)r0 * (size_t)H, rows, H, + scales.data() + r0); + lctx.wait_kernel(run); + lctx.readback(); + std::fill(got.begin(), got.end(), 0.0f); + lctx.dequant_pb16i(got.data(), rows, T, scales.data() + r0, lm_head_packed_[(size_t)t].eref, + xd_->head_ctx()->pb16i_planes); + for (int m = 0; m < rows; m++) { + for (int n = n0; n < n1; n++) { + float v = got[(size_t)m * T + (n - n0)]; + if (sem_.logit_softcap > 0) v = sem_.logit_softcap * std::tanh(v / sem_.logit_softcap); + logits_all[(size_t)(r0 + m) * (size_t)NV + (size_t)n] = v; + } } } } diff --git a/tests/npu_row_chunks_test.cpp b/tests/npu_row_chunks_test.cpp new file mode 100644 index 000000000..f933bced6 --- /dev/null +++ b/tests/npu_row_chunks_test.cpp @@ -0,0 +1,113 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// npu_row_chunks_test.cpp -- the invariant that keeps a batched verify pass inside its context. +// +// engine#1386: the pb16i LM-head rows path launched all M rows into a 16-row context; the quantiser +// refused the batch, returned the rows unquantised, and the launch ran anyway (engine#1372's failure on +// the head). The fix is the chunking #1364 shipped for the int8 head, and these are the properties that +// make it a fix rather than a rearrangement: +// +// no chunk wider than the capacity, the chunks contiguous and in order, every row covered exactly +// once, M == 0 as no chunks, an unknown capacity as no chunks (so the caller falls back instead of +// launching blind), and the A/B arm (one row per chunk) as the same coverage. +// +// Device-free: the header is arithmetic only. A device A/B of the launch count is #1386's own note. +#include +#include +#include + +#include "q4nx_row_chunks.h" + +namespace { + +int g_failures = 0; + +void check(bool cond, const char* what) { + if (!cond) { + std::printf("FAIL %s\n", what); + ++g_failures; + } +} + +// Coverage: contiguous from 0, no gaps, no overlaps, total == rows, each chunk <= capacity, all >= 1. +void check_cover(int rows, int capacity) { + const std::vector cs = onebit::npu::row_chunks(rows, capacity, true); + int next = 0; + std::set seen; + for (const auto& c : cs) { + check(c.rows >= 1, "chunk is non-empty"); + check(c.rows <= capacity, "chunk is no wider than the capacity"); + check(c.offset == next, "chunk is contiguous with the previous one"); + for (int r = c.offset; r < c.offset + c.rows; ++r) check(seen.insert(r).second, "row covered once"); + next = c.offset + c.rows; + } + check(next == rows, "chunks cover every row"); + check((int)cs.size() == (rows + capacity - 1) / capacity, "chunk count is ceil(rows / capacity)"); +} + +} // namespace + +int main() { + using onebit::npu::row_chunks; + + // 1. The regression itself: M above the 16-row twin capacity must become ceil(M/16) chunks, never one + // launch of M rows. The old code passed M straight through, which is what the device refused. + for (int m : {17, 18, 31, 32, 33, 64, 128}) check_cover(m, 16); + check((int)row_chunks(17, 16, true).size() == 2, "17 rows into a 16-row context is 2 launches"); + check((int)row_chunks(128, 16, true).size() == 8, "a 128-row verify batch is 8 launches"); + check(row_chunks(17, 16, true)[0].offset == 0 && row_chunks(17, 16, true)[1].offset == 16, + "the second chunk starts at row 16"); + check(row_chunks(17, 16, true)[1].rows == 1, "the tail chunk carries the 1 leftover row"); + + // 2. Small and exact batches are unchanged: M <= capacity is one launch of M rows (so M=16 is + // bit-for-bit what the path did before chunking). + for (int m : {1, 2, 16}) { + const auto cs = row_chunks(m, 16, true); + check((int)cs.size() == 1 && cs[0].offset == 0 && cs[0].rows == m, "M <= capacity is one launch"); + } + check((int)row_chunks(16, 16, true).size() == 1, "M == capacity is exactly one launch"); + check_cover(1, 16); + check_cover(16, 16); + + // 3. The prefill design's capacity (128 rows) is where the head would never need chunking, but the + // arithmetic must still hold for it. + check((int)row_chunks(128, 128, true).size() == 1, "128 rows into a 128-row context is one launch"); + check_cover(130, 128); + + // 4. The A/B arm (NPU_HEAD_NO_CHUNK=1): one row per chunk, same coverage - that is what the path did + // before chunking existed, so a numeric A/B compares artifacts, not coverage. + const auto per_row = row_chunks(17, 16, false); + check((int)per_row.size() == 17, "the no-chunk arm is one launch per row"); + int next = 0; + for (const auto& c : per_row) { + check(c.rows == 1 && c.offset == next, "no-chunk arm covers rows one at a time, in order"); + ++next; + } + + // 5. Refusals, not guesses: no rows and an unknown capacity both produce no chunks, so the caller + // takes its per-row path rather than launching into a context it cannot size. + check(row_chunks(0, 16, true).empty(), "0 rows is no chunks"); + check(row_chunks(-3, 16, true).empty(), "a negative row count is no chunks"); + check(row_chunks(64, 0, true).empty(), "capacity 0 is no chunks"); + check(row_chunks(64, -1, true).empty(), "a negative capacity is no chunks"); + + if (g_failures == 0) { + std::printf("npu_row_chunks: PASS\n"); + return 0; + } + std::printf("npu_row_chunks: %d failure(s)\n", g_failures); + return 1; +}