From e4d1a764dd6ce232a682cc06acb37e96851ab472 Mon Sep 17 00:00:00 2001 From: pi-inferzo Date: Tue, 10 Nov 2026 06:11:11 -0400 Subject: [PATCH 1/2] npu: the embedding guard's expected size comes from the dtype, so a quantised table is not refused MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit engine#1881 replaced the file-end bound with the tensor's own span, which was right, but kept a **bf16** figure as the expected size: if (…embed_tokens.weight").bytes < size_t(d.vocab) * size_t(d.hidden) * 2) For a quantised table those are not comparable. Measured against the container on this box: bonsai2-27b's I8 tiled embedding is 794,624,000 B against that expression's 2,542,796,800 B, so a container of exactly the kind #1807/#1878 exist to keep working would be refused with a message saying its table is smaller than vocab x hidden bf16 values -- true, and irrelevant, because the table IS smaller: it stores 0.625 B/element, not 2. The expected size now comes from the dtype (, 0 for a dtype this reader does not decode), and the comparison is made only where a row-major reading is meaningful: * a float table (bf16/f16/f32) of V x H must hold at least bytes(V*H*width) -- so the case #1881 was written for, a config overstating vocab, is still refused; * an I8 table is either plain row-major (exactly V*H bytes) or a CHUNK GRID, whose shape product is already a byte count (155,200 "rows" = 7,760 row tiles x 20 column tiles). Comparing those against V*H is the mistake, so I8 keeps the file-end bound -- looser, but never wrong on a table that is legitimately small. Verified against the three cases by arithmetic (bf16 exact ok; bf16 short refused; I8 quantised ok). NOT built locally: disk is 158 G against the 175 G floor, and lane.cpp needs the private runtime to compile, so this is CI-verified; npu/model.cpp passes -fsyntax-only and tools/copyright.py --check is clean. --- npu/lane.cpp | 19 +++++++++++++++++-- npu/model.cpp | 14 ++++++++++++++ npu/model.h | 6 ++++++ 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/npu/lane.cpp b/npu/lane.cpp index f8414ed79..43ade6a9b 100644 --- a/npu/lane.cpp +++ b/npu/lane.cpp @@ -104,8 +104,23 @@ struct Lane::Impl { throw std::runtime_error("model.q4nx: model.norm.weight holds " + std::to_string(norm.bytes) + " bytes, fewer than hidden x 2"); std::memcpy(fnorm->map(), norm.data, size_t(d.hidden) * 2); - if (model.tensor("model.embed_tokens.weight").bytes < size_t(d.vocab) * size_t(d.hidden) * 2) - throw std::runtime_error("model.q4nx: model.embed_tokens.weight holds fewer than vocab x hidden bf16 values"); + // engine#1881 bounded this by the tensor's own span (`bytes`), which is right, but kept a **bf16** expected + // size -- and that refuses a quantised table for being smaller than an f32->bf16 table it never claimed to be + // (bonsai2-27b: 794,624,000 B of I8 chunks against 2,542,796,800 B). The expected size comes from the dtype now, + // and the comparison is made only where a row-major reading is meaningful: + // * a float table (bf16/f16/f32) of V x H must be AT LEAST bytes(V*H*width) -- the case #1881 was written for, + // a config overstating vocab, still refused; + // * an I8 table is either plain row-major (exactly V*H bytes, the gemma4 layout) or a CHUNK GRID, whose shape + // product is already a byte count (155,200 "rows" are 7,760 row tiles x 20 column tiles). Comparing those + // against V*H is the mistake this guard made, so an I8 table keeps the file-end bound (`avail`), which is a + // looser but never-wrong answer. + const Tensor& emb = model.tensor("model.embed_tokens.weight"); + const uint64_t emb_need = emb.dtype == "I8" ? 0 : emb.expected_bytes(uint64_t(d.vocab) * uint64_t(d.hidden)); + const uint64_t emb_have = emb_need ? emb.bytes : emb.avail; + if (emb_need ? (emb.bytes < emb_need) : (emb_have < size_t(d.vocab) * size_t(d.hidden) * 2)) + throw std::runtime_error("model.q4nx: model.embed_tokens.weight holds " + std::to_string(emb_have) + + " bytes, fewer than the " + std::to_string(d.vocab) + " x " + std::to_string(d.hidden) + + " elements config.json declares" + (emb_need ? "" : " (I8 chunked table: file-end bound)")); lmhead_w = make_bo(dev, lmhead_weight_bytes(model)); pack_lmhead_weights(model, static_cast(lmhead_w->map())); diff --git a/npu/model.cpp b/npu/model.cpp index fb8a5fed6..f83aaf77d 100644 --- a/npu/model.cpp +++ b/npu/model.cpp @@ -29,6 +29,20 @@ namespace onebit::npu { +uint64_t Tensor::expected_bytes(uint64_t elements) const { + // Element widths of the dtypes this reader decodes. Callers pass a row-major element count, so a tiled/chunked entry + // (whose shape product is already a byte count: its last dimension is the chunk's BYTES) must not be sent here -- + // npu/lane.cpp checks `shape.size() == 2` for that reason. 0 means "not understood", and the caller then keeps the + // file-end bound rather than refusing on a guess. + uint64_t per = 0; + if (dtype == "BF16" || dtype == "F16") per = 2; + else if (dtype == "F32") per = 4; + else if (dtype == "I32") per = 4; + else if (dtype == "I8" || dtype == "U8") per = 1; + if (per == 0 || elements == 0) return 0; + return per * elements; +} + int Tensor::tiles() const { // [tile rows, tile cols, chunk bytes] (Q4NX with Q4_K or Q8 chunks, npu/q4nx.h) if (shape.size() == 3 && shape[0] > 0 && shape[1] > 0) return int(shape[0] * shape[1]); diff --git a/npu/model.h b/npu/model.h index e6b7c7a74..3d13cc53a 100644 --- a/npu/model.h +++ b/npu/model.h @@ -42,6 +42,12 @@ struct Tensor { // Tiles in a projection: shape[0] rows of shape[1] bytes, in 5120-byte tiles. int tiles() const; + + // Bytes this tensor SHOULD occupy for `elements` logical elements, from its own dtype, or 0 for a dtype this reader + // does not decode. engine#1881 bounded the embedding by the tensor's own span (`bytes`) but kept a **bf16** figure as + // the expected size, which refuses a quantised table: bonsai2-27b's I8 tiled table is 794,624,000 B against that + // expression's 2,542,796,800 B. `bytes` is the right field; the expected number has to come from the dtype. + uint64_t expected_bytes(uint64_t elements) const; }; struct Dims { From e47ee588a2a7553e6e132e70db667640492801dd Mon Sep 17 00:00:00 2001 From: pi-inferzo Date: Tue, 10 Nov 2026 14:55:05 -0400 Subject: [PATCH 2/2] test: pin Tensor::expected_bytes' widths, the table the embedding guard depends on MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The guard added in this branch is only as right as the dtype table behind it, and a quantised table is exactly the case that broke: #1881 compared the embedding's bytes against a bf16 figure, so an I8 table (0.625 B/element) was refused for being legitimately small (bonsai2-27b: 794,624,000 B against 2,542,796,800 B). Eight arms: bf16/f16 = 2, f32/i32 = 4, i8/u8 = 1, an UNKNOWN dtype answers 0 (so the caller keeps the file-end bound instead of refusing on a guess), and zero elements is 0 rather than a bogus width. Built and run locally (not CI-only any more): cmake --build build-dx --target npu_model_bounds_test -> rc 0 ./build-dx/npu_model_bounds_test -> 8/8 ok, 'expected_bytes: 8 arms …' --- tests/npu_model_bounds_test.cpp | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/npu_model_bounds_test.cpp b/tests/npu_model_bounds_test.cpp index cd4ff1921..c0acdc047 100644 --- a/tests/npu_model_bounds_test.cpp +++ b/tests/npu_model_bounds_test.cpp @@ -119,7 +119,30 @@ std::vector one_layer(std::vector q_shape = {1, 5120}, int64_t g } // namespace +// engine#1890: the widths `Tensor::expected_bytes()` uses, pinned here because the guard in npu/lane.cpp is only as right as +// this table. A quantised table is the case that broke: #1881 compared the embedding's `bytes` against a bf16 figure, so an I8 +// table (0.625 B/element) was refused for being legitimately small (bonsai2-27b: 794,624,000 B against 2,542,796,800 B). The +// table below is what makes the comparison legal, and the 0 return is what keeps an unknown dtype on the file-end bound instead +// of refusing on a guess. +void expected_bytes_widths() { + auto tb = [](const char* dtype) { + Tensor t; + t.dtype = dtype; + return t; + }; + check(tb("BF16").expected_bytes(1000) == 2000, "expected_bytes: BF16 is 2 bytes per element"); + check(tb("F16").expected_bytes(1000) == 2000, "expected_bytes: F16 is 2 bytes per element"); + check(tb("F32").expected_bytes(1000) == 4000, "expected_bytes: F32 is 4 bytes per element"); + check(tb("I32").expected_bytes(1000) == 4000, "expected_bytes: I32 is 4 bytes per element"); + check(tb("I8").expected_bytes(1000) == 1000, "expected_bytes: I8 is 1 byte per element"); + check(tb("U8").expected_bytes(1000) == 1000, "expected_bytes: U8 is 1 byte per element"); + check(tb("Q4").expected_bytes(1000) == 0, "expected_bytes: an unknown dtype answers 0, so the caller keeps `avail`"); + check(tb("BF16").expected_bytes(0) == 0, "expected_bytes: zero elements is 0, not a bogus width"); + std::printf(" expected_bytes: 8 arms (six widths, unknown dtype, zero elements)\n"); +} + int main() { + expected_bytes_widths(); const fs::path root = fs::temp_directory_path() / ("npu_model_bounds_test." + std::to_string(::getpid())); fs::remove_all(root);