diff --git a/CMakeLists.txt b/CMakeLists.txt index 2e18dcbef..d4754190d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -348,6 +348,15 @@ add_test(NAME bench_preflight COMMAND bash ${CMAKE_SOURCE_DIR}/tools/tests/test- # engine#1702: builds a per-layer MoE expert placement table from a moe_trace, and refuses a prefill-only trace because # placement is a decode lever (5.4-6.9 % on batched prefill against dedup's 90-94 %). --self-test drives it on synthetic # traces with a known answer; no model, no device. +# engine#1990: the padded-head row arithmetic, driven on the shape that broke it (CodeLlama-7b's I8 [16016, 6144] for a +# 32,032-row head at kpad 4,096). It is header-only (`npu/forward/include/pad_head_rows.h`) precisely so it can live HERE: +# an earlier version linked `onebit_npu_forward`, which exists only under `if(ONEBIT_NPU)`, so CI -- which configures +# without the NPU runtime -- never built or ran the arm. `add_dependencies(onebit ...)` keeps it in the build CI runs. +add_executable(npu_pad_head_rows_test tests/npu_pad_head_rows_test.cpp) +target_include_directories(npu_pad_head_rows_test PRIVATE npu/forward/include) +add_dependencies(onebit npu_pad_head_rows_test) +add_test(NAME npu_pad_head_rows COMMAND npu_pad_head_rows_test) + # engine#1934: reading ONE tensor out of a safetensors shard needs `data_offsets` measured from 8 + header_len, not # from byte 8. Getting that wrong returns a *different* tensor of the same size that still looks finite, and it cost # two false conclusions about a pack in one night. `--self-test` drives the rule on a synthetic shard (no network): diff --git a/npu/forward/include/pad_head_rows.h b/npu/forward/include/pad_head_rows.h new file mode 100644 index 000000000..89e0f77f7 --- /dev/null +++ b/npu/forward/include/pad_head_rows.h @@ -0,0 +1,67 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// pad_head_rows.h -- the padded-head row arithmetic, header-only so a test can drive it where CI configures. +// +// It lived as a free function in q4nx_forward.cpp, which meant the arm had to link `onebit_npu_forward` -- a library +// that only exists under `if(ONEBIT_NPU)`, so CI (which configures WITHOUT the NPU runtime) never built or ran the test. +// A regression arm nothing executes is the failure this whole file exists to avoid, so the rule moved here and the arm +// now sits outside the NPU guard. +#pragma once + +#include "q4nx_dequant.h" + +#include + +// engine#1990: how many rows to dequantise for the lm_head, given the tensor's OWN shape and the container's K grid. +// A vocabulary need not be a multiple of the tile's 32 rows (CodeLlama-7b: 32,016; MiniCPM-V 4.7 1B: 248,094), so a +// container stores the head on the next 32-row boundary and this returns that PADDED row count; the caller decodes +// that many rows and keeps NV of them. Pure, so the numbers are drivable without a container +// (tests/npu_pad_head_rows_test.cpp). +// +// Three spellings of the shape exist, and each says where the row count is: +// [tiles, tile bytes] shape[1] is a chunk size. shape[0] counts tiles, kpad/256 of them per 32-row block. +// Every imported container on the box (38 measured, review of #2026). +// [row tiles, column tiles, bytes] three dimensions, shape[1] == kpad/256. shape[0] already IS the block count, and +// need not divide by anything (FastFlowLM's own containers; MiniCPM-V 4.7 1B is +// I8 [7753, 4, 8704], engine#1788). +// [rows, kpad] shape[1] is the row width and not a chunk size. shape[0] is the row count itself. +// A width that is both the K grid and a chunk size (kpad 5,120) is read as tiles first, because that is what exists. +// +// One bound holds for all three and is what keeps a misread from becoming a long read: padding only ever reaches the +// next 32-row boundary, so an answer outside [NV, NV + 32) is not this tensor's row count. Reading [32032, 4096] as +// tiles gives 64,064, which fails the bound and falls through to the row spelling (review of #2026 at 63a47be). +// Anything the three spellings cannot place returns NV, and the caller's own dequant then refuses it by name. +inline bool head_chunk_bytes(int64_t b) { + return b == Q4NX_TILE_BYTES || b == 6144 /* q4_1f, the imported-container tile */ || b == Q4NX_CHUNK_Q4K || b == Q4NX_CHUNK_Q8; +} +inline int padded_head_rows(const int64_t* shape, int ndim, int kpad, int NV) { + if (!shape || kpad <= 0 || kpad % Q4NX_TILE_COLS != 0 || NV <= 0) return NV; + const int64_t per_block = kpad / Q4NX_TILE_COLS; // tiles per 32-row block + const auto fits = [NV](int64_t rows) { return rows >= NV && rows < int64_t(NV) + Q4NX_TILE_ROWS && rows % Q4NX_TILE_ROWS == 0; }; + if (ndim == 3) { + if (shape[0] <= 0 || shape[1] != per_block || !head_chunk_bytes(shape[2])) return NV; + const int64_t rows = shape[0] * Q4NX_TILE_ROWS; + return fits(rows) ? int(rows) : NV; + } + if (ndim != 2 || shape[0] <= 0) return NV; + if (head_chunk_bytes(shape[1]) && shape[0] % per_block == 0) { + const int64_t rows = (shape[0] / per_block) * Q4NX_TILE_ROWS; + if (fits(rows)) return int(rows); + } + if (shape[1] == kpad && fits(shape[0])) return int(shape[0]); + return NV; +} + diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 61480b018..127d48d06 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -24,6 +24,7 @@ // Numerics follow tools/q4nx_forward_ref.py so the two can be compared // element-wise. #include "q4nx_forward.h" +#include "pad_head_rows.h" // engine#1990: the padded-head row rule, header-only for its arm #include "batch_width.h" #include "q4nx_state_planes.h" // the plane-copy half of a snapshot, device-free and unit-tested #include "model/op_dispatch.h" // the optional-op fallback seam (engine#803 A3) @@ -1511,11 +1512,47 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, lm_head_f_.assign((size_t)NV * (size_t)kpad_, 0.0f); lm_w_ = kpad_; const uint8_t* lp = (const uint8_t*)model_tensor_data(mw_, ltd); - if (!lp || dequant_any(lp, (size_t)ltd->data_size, NV, kpad_, - lm_head_f_.data()) != 0) { + /* engine#1990: the container's head may carry MORE rows than the vocabulary, for the same reason a padded + * column exists -- q4_1 tiles hold 32-row multiples and a pack's vocabulary need not be one (CodeLlama's is + * 32,016). The importer pads those rows; the reader must therefore dequantise on the tensor's own grid and + * keep only the vocabulary's rows. Asking for exactly NV rows is what failed here: 32,016 is not a multiple + * of the tile's 32 rows, so dequant_any() refused a head that was perfectly well formed. + * + * The extra rows decode to zero and are never sampled: the logits vector stays NV long. */ + int head_rows = NV; + if (ltd->ndim >= 2 && kpad_ > 0 && kpad_ % Q4NX_TILE_COLS == 0) { + const int64_t per_block = kpad_ / Q4NX_TILE_COLS; // tiles per 32-row block + // engine#1990: the tile COUNT is shape[0] when the tensor is stored as [tiles, tile bytes] -- which is + // what this container does (CodeLlama-7b: I8 [16016, 6144] for a 32,032-row head at kpad_ 4,096) -- and + // it is shape[0] / per_block when it is stored as a row grid. Gating on `shape[1] == kpad_` saw only the + // second form, so a padded head in the first form kept head_rows = NV and dequant_any() refused it: + // "lm_head dequant failed" on a head that is exactly what the importer was asked to write. + // Both spellings count TILE-ROWS in shape[0] -- [rows, kpad_] and [tiles, tile bytes] -- so the number of + // 32-row blocks is shape[0] / per_block either way; only the row WIDTH in shape[1] differs. Exposed as a + // pure function because these numbers are what a test can drive without a container. + // The rule reads the whole shape: [tiles, tile bytes], [row tiles, column tiles, bytes] (FastFlowLM's + // own containers, engine#1788) and [rows, kpad_] each say where the row count is (pad_head_rows.h). + const int64_t head_shape[3] = {ltd->shape[0], ltd->shape[1], ltd->ndim >= 3 ? ltd->shape[2] : 0}; + head_rows = padded_head_rows(head_shape, ltd->ndim, kpad_, NV); + } + std::vector head_f32; + float* head_out = lm_head_f_.data(); + if (head_rows != NV) { + head_f32.assign((size_t)head_rows * (size_t)kpad_, 0.0f); + head_out = head_f32.data(); + } + if (!lp || dequant_any(lp, (size_t)ltd->data_size, head_rows, kpad_, head_out) != 0) { err_ = "lm_head dequant failed"; return false; } + if (head_rows != NV) { + for (int r = 0; r < NV; r++) + std::memcpy(&lm_head_f_[(size_t)r * (size_t)kpad_], &head_f32[(size_t)r * (size_t)kpad_], + sizeof(float) * (size_t)kpad_); + std::fprintf(stderr, + " [forward] lm head: %d row(s) stored for a vocabulary of %d; the %d padding row(s) are " + "not read (engine#1990)\n", head_rows, NV, head_rows - NV); + } fprintf(stderr, " [forward] separate lm_head.weight -> dequantized " "(%zu floats)\n", lm_head_f_.size()); } @@ -2538,6 +2575,7 @@ bool Q4nxNpuForward::gdn_apply_rows(int from, int to) { // its array launches are M-independent (a pb16i ELF runs its 16 rows whatever M is), so the marginal cost of // a row is host-side and has to be named, not inferred. Snapshots the same slots step() fills, prints the // delta of one step_batch() call when NPU_PHASE_PROF is set. Measurement only: nothing here is read back. + namespace { struct BatchPhases { const bool on = lh_on(); @@ -3260,15 +3298,21 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N, std::vector a8((size_t)K); for (int m = 0; m < M; m++) { const float* a = A + (size_t)m * (size_t)a_k; + // engine#845's pad on THIS leg too (kept across the merge: main fixed the IMAGE side of the padded-geometry + // problem -- unpadded images, row-major when blocked does not fit -- but the activation read still ran past the + // row). A padded design has K > a_k: chatglm3-6b's IM is 13,696 and its D design is K = 13,824, so reading A[k] + // for k < K runs 128 floats into the next row, or past the buffer. Measured before this: cos 0.40 -> -0.003 + // against the device while the f32 arm tracked it at 0.985-0.988; after, max|d| = 0 on all four steps. + const auto act = [a, a_k](int k) { return k < a_k ? a[(size_t)k] : 0.0f; }; float amax = 0.0f; for (int k = 0; k < K; k++) { - const float v = std::fabs(a[k]); + const float v = std::fabs(act(k)); if (std::isfinite(v) && v > amax) amax = v; } const float ascale = (amax >= 1e-12f) ? amax / 127.0f : 1.0f; const float tis = 1.0f / ascale; for (int k = 0; k < K; k++) { - float v = a[k]; + float v = act(k); if (!std::isfinite(v)) v = 0.0f; int q = (int)std::round(v * tis); a8[(size_t)k] = (int8_t)(q > 127 ? 127 : (q < -127 ? -127 : q)); @@ -5480,7 +5524,15 @@ bool Q4nxNpuForward::run_lm_head(const std::vector& h, std::vector // q4nx_pack_B_int8_blocked(): one multiplier per column, int8 codes. Chunked by vocabulary block, because a // whole head image is H x V bytes (622 MB at this vocabulary). const float inv_es = (sem_.tie_embeddings && sem_.embed_scale > 0) ? 1.0f / sem_.embed_scale : 1.0f; - const int TB = 8192; // vocabulary block; a tile's image is H x TB bytes + // engine#1990: the block width is an experiment knob, because the residual on CodeLlama is confined to exactly + // 16 logit positions starting at 16384 -- the start of a block -- and is identical at every step, i.e. a WEIGHT + // difference localised at a block boundary rather than an accumulation difference. NPU_I8_HEAD_TB moves the + // boundaries: if the wrong positions move with it, the chunking is the cause; if they stay at 16384, it is not. + static const int TB = [] { + const char* e = std::getenv("NPU_I8_HEAD_TB"); + const int v = e ? std::atoi(e) : 8192; + return (v >= 1024 && v % 128 == 0) ? v : 8192; + }(); if ((int)lm_head_i8_.size() != (NV + TB - 1) / TB) lm_head_i8_.assign((NV + TB - 1) / TB, HostI8Tensor()); logits.assign((size_t)NV, 0.0f); std::vector a8v((size_t)H); diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index d1e844bc7..5cce667b1 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -2260,6 +2260,30 @@ bool external_container_tensors(const std::string& role, int nh, int nkv, int hd return false; } +// engine#1990: q4_1 tiles hold 32-row multiples, and a pack's vocabulary need not be one. CodeLlama's is 32,016, which +// refused the whole import with "q4_1 tiles need N % 32 == 0" -- a head the grid cannot hold, not a head that is wrong. +// Where the pack pads its own head (the .fconst tail carries ceil(V/64)*64 rows: Hunyuan-0.5B has 120,832 for 120,818) +// nothing here fires; where it does not, pad with rows that decode to zero, and leave `vocab_size` at the vocabulary's +// own number so the padding is never addressed. codes 0 with SCALE 1 and zero 0 decode to exactly 0.0 -- and scale 1, +// not 0: a zero scale is not a zero weight to a reader, it is a malformed parameter block (the forward's dequant refuses +// it, which is how the first version of this padding was caught: "lm_head dequant failed"). +void pad_rows_to_tile_quantum(Q4& q, int vocab) { + if (q.N <= 0 || q.K <= 0 || q.block <= 0 || q.K % q.block) return; + if (q.N % q4nx::kRows == 0) return; + const int padded = (q.N + q4nx::kRows - 1) / q4nx::kRows * q4nx::kRows; + const size_t gpr = size_t(q.K / q.block); // groups per row + for (int r = q.N; r < padded; r++) { + q.codes.insert(q.codes.end(), size_t(q.K), 0); + q.scale.insert(q.scale.end(), gpr, 1.0f); // 1.0, not 0: code 0 * 1 + 0 = 0.0, and a 0 scale is malformed + q.zero.insert(q.zero.end(), gpr, 0.0f); + } + std::fprintf(stderr, + "note: the head is %d rows for a vocabulary of %d (padded from %lld to the 32-row tile quantum); " + "vocab_size stays %d in the container\n", + padded, vocab, (long long)q.N, vocab); + q.N = padded; +} + Q4 container_from_external_q4(const Q4& pack, long long* m_inexact) { // A PASS-THROUGH since the signed-int4 fix: `decode_external_tile` produces the container's own convention directly // (code = c_signed + 8, zero = (-8 - z_signed) * scale), so converting `zero` again applies the mapping TWICE -- @@ -4971,6 +4995,7 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons uint64_t qw_need = 0; // N * K/block * block/2 if (!mul_ok(groups_n, uint64_t(mm.block) / 2, qw_need)) throw std::runtime_error("lm_head: qweight length overflows"); Q4 q = decode_matmulnbits(bytes_of(wt, qw_need), scales.data(), zp, int(mm.N), int(mm.K), mm.block); + pad_rows_to_tile_quantum(q, int(V)); // no qzeros tensor -> the implicit zero point (engine#1503: Qwen3-VL-4B's spelling, three-input nodes) if (!have_zp) implicit_zp_zero(q, mm.bits); diff --git a/npu/pack_import.h b/npu/pack_import.h index 513025dbf..bf3b4ff5b 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -641,6 +641,11 @@ constexpr bool kExtQuantsRowMajor = false; // measurably NOT the same weights: on a real phi-4 tile it reproduced the pack's codes on only 2,826 of 8,192 // entries and moved weights by 2.5 % of the tensor's range (engine#1503). `m_inexact` reports how many group // offsets were not bf16-exact (86.3 % are, measured), because that rounding is the only loss this path has. +// engine#1990: a head whose row count is not a multiple of the 32-row tile quantum (CodeLlama: 32,016 rows) cannot be +// held by the q4_1 grid, so pad it with zero rows that decode to zero and leave the container's vocab_size alone. Exposed +// because the property -- padding is invisible to the vocabulary -- is what a test can pin without a pack. +void pad_rows_to_tile_quantum(Q4& q, int vocab); + Q4 container_from_external_q4(const Q4& pack, long long* m_inexact = nullptr); // The container role a matmul plays, read off the graph's own tensor name: `...qkv_proj...` -> "QKV", diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 57e3e8862..0fe5256f4 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -1404,6 +1404,53 @@ void test_1671_plan_head_arm(const std::string& root) { } // ── 1. MatMulNBits: encode here, decode with the importer ──────────────────── +// engine#1990: a head whose row count is not a multiple of the 32-row tile quantum (CodeLlama's vocabulary is 32,016) is +// padded with rows that decode to zero, and the container's vocab_size is NOT changed -- the padding is invisible to the +// vocabulary. Both halves are pinned here because widening N without staying invisible is the failure that would show up +// as two plausible extra tokens at the end of every distribution. +void test_head_row_padding() { + const auto make = [](int N) { + pi::Q4 q; + q.N = N; + q.K = 128; // two 64-column tiles + q.block = 32; // four groups per row + q.codes.assign(size_t(N) * size_t(q.K), 1); // non-zero, so padding is distinguishable + q.scale.assign(size_t(N) * size_t(q.K / q.block), 0.5f); // 1.0 in the padded rows; see below + q.zero.assign(size_t(N) * size_t(q.K / q.block), 0.25f); + return q; + }; + { + pi::Q4 q = make(32016); + pi::pad_rows_to_tile_quantum(q, 32016); + check(q.N == 32032 && q.N % 32 == 0, "head padding: 32,016 rows become 32,032 (the 32-row quantum)"); + check(q.codes.size() == size_t(32032) * 128 && q.scale.size() == size_t(32032) * 4, + "head padding: codes and scales grow with the rows, one group's worth each"); + bool zeros = true; + for (int r = 32016; r < q.N; r++) { + for (int k = 0; k < q.K; k++) zeros = zeros && q.codes[size_t(r) * 128 + size_t(k)] == 0; + for (int g = 0; g < 4; g++) + zeros = zeros && q.scale[size_t(r) * 4 + size_t(g)] == 1.0f && q.zero[size_t(r) * 4 + size_t(g)] == 0.0f; + } + check(zeros, "head padding: every padded row is code 0, scale 1, zero 0 -- which decodes to 0.0, not to a weight"); + bool kept = true; + for (int r = 0; r < 32016; r++) + for (int k = 0; k < 128; k++) kept = kept && q.codes[size_t(r) * 128 + size_t(k)] == 1; + check(kept, "head padding: the vocabulary's own rows are untouched"); + } + { // an aligned vocabulary is left exactly as it was: no note, no growth + pi::Q4 q = make(32000); + const size_t before = q.codes.size(); + pi::pad_rows_to_tile_quantum(q, 32000); + check(q.N == 32000 && q.codes.size() == before, "head padding: a 32-aligned vocabulary (32,000) is not touched"); + } + { // 16 rows of slack and 1 row of slack both round up to one tile boundary, not two + pi::Q4 a = make(32017), b = make(32031); + pi::pad_rows_to_tile_quantum(a, 32017); + pi::pad_rows_to_tile_quantum(b, 32031); + check(a.N == 32032 && b.N == 32032, "head padding: 32,017 and 32,031 both round to 32,032"); + } +} + void test_matmulnbits() { const int N = 64, K = 256, B = 128; std::vector qw(size_t(N) * (K / B) * (B / 2)); @@ -5706,7 +5753,8 @@ int main(int argc, char** argv) { test_crafted_refusals(golden_dir.empty() ? "." : golden_dir); test_modelbin_genre_refusal(golden_dir.empty() ? "." : golden_dir); test_1671_plan_head_arm(golden_dir.empty() ? "." : golden_dir); - test_matmulnbits(); + test_head_row_padding(); + test_matmulnbits(); test_tiles(golden_dir); test_q4_1f(); test_attn_tile(golden_dir); diff --git a/tests/npu_pad_head_rows_test.cpp b/tests/npu_pad_head_rows_test.cpp new file mode 100644 index 000000000..23dcaa042 --- /dev/null +++ b/tests/npu_pad_head_rows_test.cpp @@ -0,0 +1,148 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// npu_pad_head_rows_test.cpp -- how many rows the padded-head path dequantises, driven on the numbers that broke it. +// +// engine#1990 pads the lm_head to the q4_1 tile quantum at import (a vocabulary need not be a multiple of the tile's 32 +// rows: CodeLlama-7b's is 32,016 -> 32,032) and the reader then has to dequantise the TENSOR's grid while keeping only +// the vocabulary's rows. The first version gated that on `shape[1] == kpad_`, which is true for a `[rows, kpad_]` head and +// false for the `[tiles, tile bytes]` head the importer writes -- so it fell back to `NV` rows and `dequant_any()` refused a +// well-formed head with "lm_head dequant failed". CodeLlama-7b: `I8 [16016, 6144]`, kpad 4096, NV 32,016. +// +// The arithmetic is pure, so every case here runs with no container, no device and no pack. +#include "pad_head_rows.h" // the rule, header-only: no library, so CI can build this + +#include +#include +#include + + +static int failures = 0; + +static void check(bool ok, const char* what, long long got, long long want) { + if (ok) { + std::printf("ok %s\n", what); + return; + } + std::printf("FAIL %s (got %lld, want %lld)\n", what, got, want); + ++failures; +} + +static int rows2(int64_t s0, int64_t s1, int kpad, int NV) { + const int64_t shape[3] = {s0, s1, 0}; + return padded_head_rows(shape, 2, kpad, NV); +} +static int rows3(int64_t s0, int64_t s1, int64_t s2, int kpad, int NV) { + const int64_t shape[3] = {s0, s1, s2}; + return padded_head_rows(shape, 3, kpad, NV); +} + +int main() { + // The shape this path exists for: 16,016 tiles of 6,144 B, K grid 4,096, vocabulary 32,016. + { + const int r = rows2(16016, 6144, 4096, 32016); + check(r == 32032, "the [tiles, tile bytes] head dequantises its PADDED grid (32,016 -> 32,032)", r, 32032); + check(r % Q4NX_TILE_ROWS == 0, "and that grid is whole 32-row tiles", r % Q4NX_TILE_ROWS, 0); + } + // The same head in the 5,120-byte tile, as codellama7b-b8 stores it. + { + const int r = rows2(16016, 5120, 4096, 32016); + check(r == 32032, "the same head in 5,120-byte tiles gives the same rows", r, 32032); + } + // The ROW spelling of that head (review of #2026 at 63a47be): the tensor has 32,032 rows of 4,096, so shape[0] is + // the row count itself. Read as tiles it would be 64,064, twice what the tensor holds. + { + const int r = rows2(32032, 4096, 4096, 32016); + check(r == 32032, "the [rows, kpad] spelling reads shape[0] as rows, not as tiles", r, 32032); + } + // ... and a row-spelled head that is NOT padded to the next boundary is not a padded head. + { + const int r = rows2(32064, 4096, 4096, 32016); + check(r == 32016, "a row grid more than one block past the vocabulary falls back to NV", r, 32016); + } + // The three-dimensional spelling, FastFlowLM's own (engine#1788): [row tiles, column tiles, tile bytes]. 7,753 is odd, + // so no rule that divides shape[0] by the 4 column tiles can place it. + { + const int r = rows3(7753, 4, 8704, 1024, 248094); + check(r == 248096, "MiniCPM-V 4.7 1B's [7753, 4, 8704] head is 248,096 stored rows for 248,094", r, 248096); + } + { + const int r = rows3(7755, 8, 8704, 2048, 248144); + check(r == 248160, "MiniCPM-V 4.7 35B-A3B's [7755, 8, 8704] head is 248,160 stored rows for 248,144", r, 248160); + } + // A three-dimensional head whose column tiles are not this K grid's is some other tensor: nothing is assumed. + { + const int r = rows3(7753, 8, 8704, 1024, 248094); + check(r == 248094, "a 3-D head with the wrong column-tile count falls back to NV", r, 248094); + } + { + const int r = rows3(7753, 4, 1000, 1024, 248094); + check(r == 248094, "a 3-D head whose last dimension is not a tile size falls back to NV", r, 248094); + } + // A width that is both the K grid and a tile size (kpad 5,120, bonsai2-27b: [155200, 5120] for 248,320 rows) is tiles. + { + const int r = rows2(155200, 5120, 5120, 248320); + check(r == 248320, "a 5,120-wide K grid in 5,120-byte tiles is read as tiles and needs no padding", r, 248320); + } + // A head that needs no padding keeps the vocabulary's own count. + { + const int r = rows2(16000, 6144, 4096, 32000); + check(r == 32000, "an exactly-aligned vocabulary stays at NV", r, 32000); + } + // A K grid that is not a whole number of tile columns cannot be walked, so NV is the honest answer (the caller then + // refuses, which is what happened before the fix -- and is correct for a grid this function does not understand). + { + const int r = rows2(16016, 6144, 4095, 32016); + check(r == 32016, "a kpad that is not whole tile columns falls back to NV", r, 32016); + } + // A tensor narrower than the vocabulary is not a padded head; the caller must not be told to read past its rows. + { + const int r = rows2(500, 6144, 4096, 32016); + check(r == 32016, "a grid smaller than NV falls back to NV rather than a short read", r, 32016); + } + // The LOWER end of the bound, in each spelling (review of #2026 at 3a1b03a). The arm above is declined because 500 is + // not whole blocks, not because it is short: these three are whole 32-row grids, so only `rows >= NV` stands between + // them and a dequant of 1,024 rows for a vocabulary of 32,016. Deleting that comparison fails all three. + { + const int r = rows2(512, 6144, 4096, 32016); + check(r == 32016, "a whole-block tile grid short of the vocabulary is declined by the bound, not answered with 1,024", r, 32016); + } + { + const int r = rows3(32, 16, 6144, 4096, 32016); + check(r == 32016, "a 3-D grid short of the vocabulary is declined by the bound, not answered with 1,024", r, 32016); + } + { + const int r = rows2(1024, 4096, 4096, 32016); + check(r == 32016, "a row grid short of the vocabulary is declined by the bound, not answered with 1,024", r, 32016); + } + // A partial block (shape[0] not a multiple of tiles-per-block) is not a grid this function can walk. + { + const int r = rows2(16001, 6144, 4096, 32016); + check(r == 32016, "a partial block falls back to NV", r, 32016); + } + // More than one block of padding is not padding: a tile grid twice the vocabulary must not be read. + { + const int r = rows2(32032, 6144, 4096, 32016); + check(r == 32016, "a tile grid far past the vocabulary falls back to NV", r, 32016); + } + // NV <= 0 is a caller bug, and returning NV keeps the failure at the caller's check rather than here. + { + const int r = rows2(16016, 6144, 4096, 0); + check(r == 0, "a non-positive vocabulary is passed through, not invented", r, 0); + } + + if (failures == 0) std::printf("all padded-head row checks passed\n"); + return failures == 0 ? 0 : 1; +}