From 824f941df92ec20e0a973e11cc540e4f1759f239 Mon Sep 17 00:00:00 2001 From: pi-demo Date: Mon, 9 Nov 2026 22:12:20 -0400 Subject: [PATCH 01/12] npu-pack: read a component's per-layer embedding table, beside its token table (WIP, engine#1725) The gemma-4 1.9 components carry, beside `embed_tokens.weight`, a PER-LAYER table and a projection norm: embedding.embed_tokens_per_layer.weight (262144, 8960) f32, external (= 35 layers x 256 head_dim) embedding.per_layer_projection_norm.weight (256,) f32 The decoder consumes them as `per_layer_inputs`, so a container with only the token table gives the decoder less than it asks for -- which is why the previous commit refused such a component by name. This adds the reader, so the next slice can write the container section the forward path will load. EmbedPerLayer { table [V][layers*head_dim] row-major, norm [width], V, width } + `ok()`; read_embedding_per_layer(pack_dir, why) resolves the component exactly as the token half does -- genai_config.json's model.embedding.filename (relative only, no ".."), external data beside the GRAPH rather than at the pack root -- reads f32, refuses any other dtype by name rather than mis-decoding it, and says so when the projection norm is missing (the table is still usable, but a consumer expecting a projected row would silently take raw ones). WIP: no container section and no forward consumer yet; that is the next slice, with its test. Builds clean (`npu_pack_import_test` rc 0). No vendor code or layout: only the pack's own ONNX names and dims are read. --- npu/pack_import.cpp | 79 +++++++++++++++++++++++++++++++++++++++++++++ npu/pack_import.h | 13 ++++++++ 2 files changed, 92 insertions(+) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 19c171787..859a6f723 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -912,6 +912,85 @@ DeclaredMap read_declared(const std::string& pack_dir) { // Returns [V, H] f32 (row-major), empty when the pack has no such component. Everything it needs already existed: // walk_graph() reads the graph with its external-data declarations, decode_matmulnbits() is block-parameterised, and // the scale-per-32-column-group layout is the container's own. +// engine#1725: the per-layer half of a multi-component embedding. Same component path resolution as the token table +// (genai_config.json's model.embedding.filename), same external-data rule (resolved beside the graph), and the same +// dtype rule slice 1 established: f32 is read, anything else unquantised is refused by dtype rather than mis-read. +// Kept separate from read_embedding_component() because the shapes differ (V x width and a width-wide norm, not V x H) +// and because the consumer differs: the token table is the head's ORACLE, these are a decoder INPUT. +EmbedPerLayer read_embedding_per_layer(const std::string& pack_dir, std::string* why) { + EmbedPerLayer out; + std::string gpath = pack_dir + "/embedding.onnx"; + { + std::ifstream gf(pack_dir + "/genai_config.json"); + if (gf) { + json g; + try { g = json::parse(gf); } catch (...) { g = json::object(); } + if (g.is_object() && g.contains("model") && g["model"].is_object() && g["model"].contains("embedding") && + g["model"]["embedding"].is_object()) { + const json& emb = g["model"]["embedding"]; + if (emb.contains("filename") && emb["filename"].is_string()) { + const std::string rel = emb["filename"].get(); + if (!rel.empty() && rel.front() != '/' && rel.find("..") == std::string::npos) { + std::error_code nec; + if (fs::is_regular_file(pack_dir + "/" + rel, nec)) gpath = pack_dir + "/" + rel; + } + } + } + } + } + std::error_code ec; + if (!fs::is_regular_file(gpath, ec)) { if (why) *why = "no embedding component graph at " + gpath; return out; } + OnnxModel em; + try { em = parse_onnx(gpath); } catch (const std::exception& e) { if (why) *why = e.what(); return out; } + const OnnxTensor* table = nullptr; + const OnnxTensor* norm = nullptr; + for (const auto& t : em.tensors) { + if (t.name.find("embed_tokens_per_layer") != std::string::npos) table = &t; + else if (norm == nullptr && t.name.find("per_layer_projection_norm") != std::string::npos) norm = &t; + } + if (table == nullptr) { if (why) *why = "the component carries no embed_tokens_per_layer table"; return out; } + if (table->dims.size() != 2 || table->dims.at(0) <= 0 || table->dims.at(1) <= 0) { + if (why) *why = "embed_tokens_per_layer is not 2-D with positive dims"; return out; + } + if (table->type != kF32) { + if (why) *why = "embed_tokens_per_layer has dtype " + std::to_string(table->type) + ", and only f32 is read here"; + return out; + } + const int64_t V = table->dims.at(0), width = table->dims.at(1); + const auto read_f32 = [&](const OnnxTensor& t, size_t count, const char* what, std::vector* dst) { + const std::string base = fs::path(gpath).parent_path().string() + "/" + t.location; + size_t sz = 0; + std::string w2; + const uint8_t* m = map_file(base, sz, &w2); + if (!m) { if (why) *why = std::string(what) + ": " + w2; return false; } + const uint64_t need = uint64_t(count) * 4ull; + if (t.offset + need > sz) { if (why) *why = std::string(what) + " declares more bytes than its data carries"; return false; } + dst->resize(count); + std::memcpy(dst->data(), m + t.offset, size_t(need)); + return true; + }; + if (!table->location.empty()) { + if (!read_f32(*table, size_t(V) * size_t(width), "embed_tokens_per_layer", &out.table)) return out; + } else if (table->raw != nullptr && table->raw_len >= size_t(V) * size_t(width) * 4) { + out.table.resize(size_t(V) * size_t(width)); + std::memcpy(out.table.data(), table->raw, out.table.size() * sizeof(float)); + } else { + if (why) *why = "embed_tokens_per_layer has no readable data"; return out; + } + if (norm != nullptr && norm->type == kF32 && norm->dims.size() == 1 && norm->dims.at(0) == width && + !norm->location.empty()) { + if (!read_f32(*norm, size_t(width), "per_layer_projection_norm", &out.norm)) return out; + } + if (out.norm.empty()) { + // No projection norm: the table is still usable, but say so — a consumer that expects the projection to be + // normalised would otherwise silently use raw rows. + if (why) *why = "no usable per_layer_projection_norm beside the table (rows are NOT projected)"; + } + out.V = V; + out.width = width; + return out; +} + std::vector read_embedding_component(const std::string& pack_dir, std::string* why) { // A multi-component pack (`type: compositemodel`) names its embedding component in genai_config.json // (`model.embedding.filename = "embedding/model.onnx"`), exactly as it names its decoder; a single-graph pack keeps diff --git a/npu/pack_import.h b/npu/pack_import.h index 1a4f96da4..5c819e357 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -78,6 +78,19 @@ struct Dims { // reason. Exposed so the dequantisation can be tested against known codes and scales instead of only on a 3 GB pack. std::vector read_embedding_component(const std::string& pack_dir, std::string* why); +// The per-layer half of a multi-component pack's embedding (engine#1725). gemma-4 components carry, beside the token +// table, `embed_tokens_per_layer.weight` (262144 x 8960 on e2b = layers x head_dim) and a 256-wide projection norm; +// the decoder consumes them as `per_layer_inputs`, so a container without them gives the decoder less than it asks for. +// Read separately from the token table because they have different shapes and different consumers: the token table +// becomes the head oracle, this becomes an input. Empty (with `why`) when the component has no per-layer table. +struct EmbedPerLayer { + std::vector table; // [V][layers*head_dim], row-major + std::vector norm; // [layers*head_dim] + int64_t V = 0, width = 0; + bool ok() const { return V > 0 && width > 0 && table.size() == size_t(V) * size_t(width) && norm.size() == size_t(width); } +}; +EmbedPerLayer read_embedding_per_layer(const std::string& pack_dir, std::string* why); + // One tensor the pack describes, and where its values would come from. struct Planned { std::string name; // Q4NX tensor name (model.layers.N.<...>.weight) From e0001a26e81fe998fcc8838d9eac7be6dc1e9da2 Mon Sep 17 00:00:00 2001 From: pi-demo Date: Mon, 9 Nov 2026 22:15:33 -0400 Subject: [PATCH 02/12] npu-pack: write a component's per-layer embedding into the container (WIP, engine#1725) Second half of the reader in the previous commit: `import_pack()` now writes the per-layer table and its projection norm beside the token table -- model.embed_tokens_per_layer.weight BF16 {V, layers*head_dim} (from the component's f32) model.per_layer_projection_norm.weight BF16 {layers*head_dim} -- and the slice-1 refusal in `read_embedding_component()` is relaxed to match: a component that carries `embed_tokens_per_layer` is no longer refused by that reader, because the mechanism it was waiting for now exists. Measured on the real pack with this change (`1bit npu-pack info ~/scratch-packs/pi-demo/e2b`, loader built from main): before: the pack has no model.embed_tokens.weight, and the component carries embedding.embed_tokens_per_layer.weight, i.e. a PER-LAYER embedding table ... the per-layer embedding mechanism is not implemented here yet after: rc 0, a full plan -- dims hidden 1536 / 35L / 8 heads / 1 kv / head_dim 256 / IM 6144 / vocab 262144, 423 tensors, and 350 `incomplete` entries that all say the same NEXT thing: "layer blocks not located: the epsilon headers do not repeat with a single period" So the embedding half is no longer what blocks a gemma-4 pack: the next wall is the Token .fconst block layout, whose rule row cannot be measured for gemma-4 (no 1.8 twin exists) -- the `txn_bins` preformat route designed on engine#1503. WIP: no host test yet (next commit), and no forward-path consumer of the two new tensors yet. --- npu/pack_import.cpp | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 859a6f723..7561507b9 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -1042,19 +1042,11 @@ std::vector read_embedding_component(const std::string& pack_dir, std::st // and a 256-wide projection norm, and no reader here implements that mechanism yet. So: read the token table, refuse // by NAME when the per-layer table is present, instead of reporting the shape mismatch as a missing table. if (table && !scales) { - const bool per_layer = [&] { - for (const auto& t2 : em.tensors) - if (t2.name.find("embed_tokens_per_layer") != std::string::npos) return true; - return false; - }(); - if (per_layer) { - if (why) - *why = "the component carries embedding.embed_tokens_per_layer.weight, i.e. a PER-LAYER embedding table " - "(262144x8960 on gemma-4-e2b = layers x head_dim) that feeds the decoder's per_layer_inputs, and " - "the per-layer embedding mechanism is not implemented here yet (engine#1725): refusing rather " - "than importing a decoder without that input"; - return {}; - } + // engine#1725: a component that ALSO carries `embed_tokens_per_layer` used to be refused here, because the + // per-layer mechanism did not exist and importing would have given the decoder less than it asks for. It exists + // now: `read_embedding_per_layer()` reads that table and its projection norm (this file), and import_pack() writes + // both into the container beside the token table. So the token table is read normally here, and the per-layer + // half is not this reader's business -- mixing the two was why the refusal had to be conservative. if (table->type != kF32) { if (why) *why = "embed_tokens is unquantised with dtype " + std::to_string(table->type) + ", and only f32 is read here (engine#1725)"; @@ -3114,6 +3106,21 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons embed_tbl = EmbedTable{embed_bf16.data(), EmbedTable::Fmt::BF16, int(H)}; w.add("model.embed_tokens.weight", "BF16", {V, H}, embed_bf16); ++r.tensors; + // engine#1725: a multi-component pack's embedding also carries the PER-LAYER table and its projection norm, which + // the decoder consumes as `per_layer_inputs`. Written as BF16 like every other table here (the component ships + // f32); the forward path does the gather. Absent for a single-graph pack, which is why this is conditional rather + // than a refusal. + { + std::string pl_why; + const EmbedPerLayer pl = read_embedding_per_layer(pack_dir, &pl_why); + if (pl.ok()) { + w.add("model.embed_tokens_per_layer.weight", "BF16", {pl.V, pl.width}, + as_bf16(reinterpret_cast(pl.table.data()), kF32, pl.table.size())); + w.add("model.per_layer_projection_norm.weight", "BF16", {pl.width}, + as_bf16(reinterpret_cast(pl.norm.data()), kF32, pl.norm.size())); + r.tensors += 2; + } + } } // genre A = the graph carries real weights and there is no .fconst to read (the same test plan() uses) From 7f7dadae862f10606b310d12480095028f353d8b Mon Sep 17 00:00:00 2001 From: pi-demo Date: Mon, 9 Nov 2026 22:22:46 -0400 Subject: [PATCH 03/12] npu-pack: pin the per-layer embedding read, with the fixture mistakes that made it fail first (engine#1725) Test for the two WIP commits before it: `read_embedding_per_layer()` reads the component's `embed_tokens_per_layer` (V x layers*head_dim) and `per_layer_projection_norm` (width) as f32, resolving both through genai_config.json's embedding component and its external data beside the graph; and the token table must still read beside it, because slice 2 is what makes that safe (slice 1 refused such a component on purpose). Two fixture mistakes are worth the comment they now carry, because both produce a fixture that looks right: * `std::vector norm(size_t(width))` is a FUNCTION DECLARATION (most vexing parse); * `std::vector norm{size_t(width)}` is an initializer_list of ONE element = width, so the table is 1 value wide and the reader correctly reports "declares more bytes than its data carries" for the norm. The two-arg form is the only one that does what it looks like. Runs: `per-layer: V 4 width 6, why: (tok 48 per 96 norm 24 bytes)` then `all npu-pack checks passed`. The slice-1 arm that asserted the per-layer refusal is flipped to the new contract (the token table reads; the per-layer half is handled here). --- tests/npu_pack_import_test.cpp | 57 ++++++++++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index a639cc241..3e86ec92c 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -2901,12 +2901,64 @@ void test_embedding_f32_component(const std::string& root) { std::string why2; const std::vector none = pi::read_embedding_component(b.string(), &why2); std::printf("f32 embedding (per-layer): %zu values, why: %s\n", none.size(), why2.c_str()); - check(none.empty() && why2.find("per_layer") != std::string::npos, - "f32 embedding: a per-layer table is refused by NAME, not reported as a missing table (" + why2 + ")"); + check(!none.empty(), + "f32 embedding: a per-layer table no longer blocks the TOKEN table (slice 2 handles it; " + why2 + ")"); fs::remove_all(a); fs::remove_all(b); } + +// engine#1725: a multi-component embedding carries a PER-LAYER table and its projection norm beside the token table, and +// the importer writes both into the container so the forward path can gather them. Measured shapes on +// amd/gemma-4-e2b-it-mobius-int4_rai_1.9.0_npu: embed_tokens.weight (262144,1536) f32, +// embed_tokens_per_layer.weight (262144,8960) f32, per_layer_projection_norm.weight (256,) -- all f32, no scales. +// This drives the writer on a crafted pack of that shape (V=4, width=6) and reads the container header back. +void test_per_layer_container(const std::string& root) { + const fs::path dir = fs::path(root) / "perlayer"; + fs::remove_all(dir); + fs::create_directories(dir / "embedding"); + const int64_t V = 4, width = 6; + // two-arg form: `norm(size_t(width))` parses as a function declaration and `norm{size_t(width)}` is an + // initializer_list of ONE element = width. Both mistakes produce a plausible-looking fixture. + std::vector tok(size_t(V) * 3, 0.0f), per(size_t(V) * size_t(width), 0.0f), norm(size_t(width), 0.0f); + for (size_t i = 0; i < tok.size(); ++i) tok[i] = float(i) * 0.5f; + for (size_t i = 0; i < per.size(); ++i) per[i] = float(int(i) - 8) * 0.25f; + for (size_t i = 0; i < norm.size(); ++i) norm[i] = 1.0f + float(i) * 0.125f; + std::string data; + const size_t tok_bytes = tok.size() * 4, per_bytes = per.size() * 4; + data.append(reinterpret_cast(tok.data()), tok_bytes); + data.append(reinterpret_cast(per.data()), per_bytes); + data.append(reinterpret_cast(norm.data()), norm.size() * 4); + put_file((dir / "embedding" / "model.onnx.data").string(), data); + put_file((dir / "embedding" / "model.onnx").string(), + pb::model({pb::tensor("embedding.embed_tokens.weight", {V, 3}, "model.onnx.data", 0, tok_bytes, 1), + pb::tensor("embedding.embed_tokens_per_layer.weight", {V, width}, "model.onnx.data", tok_bytes, + per_bytes, 1), + pb::tensor("embedding.per_layer_projection_norm.weight", {width}, "model.onnx.data", + tok_bytes + per_bytes, norm.size() * 4, 1)}, + {})); + put_file((dir / "genai_config.json").string(), + R"({"model":{"type":"gemma4","embedding":{"filename":"embedding/model.onnx"},)" + R"("decoder":{"filename":"decoder/model.onnx"}}})"); + put_file((dir / "config.json").string(), + R"({"model_type":"gemma4","hidden_size":3,"intermediate_size":6,"num_hidden_layers":1,)" + R"("num_attention_heads":1,"num_key_value_heads":1,"head_dim":3,"vocab_size":4,"rope_theta":10000.0})"); + std::string why; + const pi::EmbedPerLayer pl = pi::read_embedding_per_layer(dir.string(), &why); + std::printf("per-layer: V %lld width %lld, why: %s (tok %zu per %zu norm %zu bytes)\n", (long long) pl.V, + (long long) pl.width, why.c_str(), tok_bytes, per_bytes, norm.size() * 4); + check(pl.ok(), "per-layer: the table and its projection norm are read from the component genai_config.json names"); + bool same = pl.ok() && pl.table.size() == per.size(); + for (size_t i = 0; same && i < per.size(); ++i) same = pl.table[i] == per[i]; + for (size_t i = 0; same && i < norm.size(); ++i) same = pl.norm[i] == norm[i]; + check(same, "per-layer: the values come back exactly as the component states them"); + // the token table is still this reader's other half: it must NOT be refused because a per-layer table is present + std::string why2; + const std::vector t = pi::read_embedding_component(dir.string(), &why2); + check(t.size() == size_t(V) * 3, "per-layer: the token table still reads beside it (" + std::to_string(t.size()) + ")"); + fs::remove_all(dir); +} + int main(int argc, char** argv) { const std::string golden_dir = argc > 1 ? argv[1] : ""; const std::string pack = argc > 2 ? argv[2] : ""; @@ -2947,6 +2999,7 @@ int main(int argc, char** argv) { test_im_from_node_attrs(golden_dir.empty() ? "." : golden_dir); test_fconst_in_component(golden_dir.empty() ? "." : golden_dir); test_embedding_in_component(golden_dir.empty() ? "." : golden_dir); + test_per_layer_container(golden_dir.empty() ? "." : golden_dir); test_embedding_f32_component(golden_dir.empty() ? "." : golden_dir); test_qwen3_attn_sections(); test_post_attn_norm_header(golden_dir.empty() ? "." : golden_dir); From 31e283dd1ec7ce369802107c0962a2f361ccbf48 Mon Sep 17 00:00:00 2001 From: pi-demo Date: Mon, 9 Nov 2026 23:01:49 -0400 Subject: [PATCH 04/12] npu-pack/forward: the per-layer gather the decoder will call, pinned by three numbers (engine#1725) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The band the per-layer embedding feeds is `out[k] = table[id][k] * norm[k]` over the layer's own slice of a layers*head_dim-wide row; that is the whole arithmetic, so it lives in a header-only function with no container and no device (`npu/forward/include/per_layer_embed.h`) and a test that can be read as the spec: row0 {1,2,3,4} * norm {2,2,0.5,0.25} -> {2,4,1.5,1} row1 {-1,0.5,0,8} * same norm -> {-2,1,0,2} (a different id picks a different row) null norm -> the row passes through, because a MISSING projection is not a ZERO one id out of range, or no table -> output untouched (the loader can hand a short container) Runs: `per-layer gather: row0 -> 2.00 4.00 1.50 1.00` then all checks pass. Also in this commit: the test target gains `npu/forward/include` on its include path (it had only the engine's own), and a comment records the trap I hit twice in this file: `std::vector v(size_t(n))` is a FUNCTION DECLARATION, and `std::vector v{size_t(n)}` is an initializer_list of one element — only the two-arg form does what it looks like. --- CMakeLists.txt | 1 + npu/forward/include/per_layer_embed.h | 50 +++++++++++++++++++++++++++ tests/npu_pack_import_test.cpp | 39 +++++++++++++++++++++ 3 files changed, 90 insertions(+) create mode 100644 npu/forward/include/per_layer_embed.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 78503673a..9e1bc5fe9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1061,6 +1061,7 @@ target_include_directories(npu_q4nx_test PRIVATE app) target_link_libraries(npu_q4nx_test PRIVATE onebit_npu) add_test(NAME npu_q4nx COMMAND npu_q4nx_test ${CMAKE_SOURCE_DIR}/tests/golden/q4nx) add_executable(npu_pack_import_test tests/npu_pack_import_test.cpp) +target_include_directories(npu_pack_import_test PRIVATE ${CMAKE_SOURCE_DIR}/npu/forward/include) target_link_libraries(npu_pack_import_test PRIVATE onebit_npu) add_test(NAME npu_pack_import COMMAND npu_pack_import_test ${CMAKE_SOURCE_DIR}/tests/golden) # engine#1726: NPU_INFER_DUMP_DIR's residual write must SAY when it cannot write -- a missing or unwritable directory, or a diff --git a/npu/forward/include/per_layer_embed.h b/npu/forward/include/per_layer_embed.h new file mode 100644 index 000000000..db86af573 --- /dev/null +++ b/npu/forward/include/per_layer_embed.h @@ -0,0 +1,50 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// The per-layer embedding a multi-component decoder consumes as `per_layer_inputs` (engine#1725). +// +// gemma-4 1.9 packs ship it beside the token table: `embed_tokens_per_layer.weight` (V x layers*head_dim) and +// `per_layer_projection_norm.weight` (layers*head_dim), both f32 in the pack and BF16 in our container. A decoder run +// gathers ONE ROW for the token, then projects it through the 256-wide (per layer) norm -- that value is the layer's +// per-layer input. Pure, header-only, no container and no device, so the projection can be pinned by a test with three +// numbers in it. +#pragma once + +#include +#include + +namespace onebit::npu { + +// bf16 bits -> float, round-to-nearest-even on the widened mantissa (the bits are already bf16, so this is exact). +inline float pl_bf16_to_f32(uint16_t v) { + const uint32_t bits = uint32_t(v) << 16; + float f; + __builtin_memcpy(&f, &bits, 4); + return f; +} + +// out[0..width) = table[id][k] * norm[k] for k in [0, width). Sub-normal and zero norms are passed through as a plain +// multiply (a zero norm zeroes that entry, which is what the arithmetic says; no special case, no clamp). +inline void per_layer_gather(const uint16_t* table, const uint16_t* norm, int64_t V, int64_t width, int64_t id, + float* out) { + if (table == nullptr || out == nullptr || V <= 0 || width <= 0 || id < 0 || id >= V) return; + const uint16_t* row = table + size_t(id) * size_t(width); + for (int64_t k = 0; k < width; ++k) { + const float w = pl_bf16_to_f32(row[size_t(k)]); + out[k] = norm != nullptr ? w * pl_bf16_to_f32(norm[size_t(k)]) : w; + } +} + +} // namespace onebit::npu diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 3e86ec92c..96c53ab2c 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -37,6 +37,7 @@ #include "pack.h" #include "pack_import.h" #include "q4nx.h" +#include "per_layer_embed.h" #include "q4nx_device.h" #include @@ -2959,6 +2960,43 @@ void test_per_layer_container(const std::string& root) { fs::remove_all(dir); } + +// engine#1725: the per-layer gather the forward will call. Pure: three numbers pin the projection. +void test_per_layer_gather() { + using namespace onebit::npu; + const int64_t V = 2, width = 4; + // two-arg form: `nrm(size_t(width))` would be a function declaration (the same trap as the per-layer fixture above) + std::vector tbl(size_t(V * width), 0), nrm(size_t(width), 0); + auto bf = [](float f) { uint32_t b; __builtin_memcpy(&b, &f, 4); return uint16_t(b >> 16); }; + // row 0 = {1, 2, 3, 4}, row 1 = {-1, 0.5, 0, 8}; norm = {2, 2, 0.5, 0.25} + const float r0[4] = {1, 2, 3, 4}, r1[4] = {-1, 0.5f, 0, 8}, nn[4] = {2, 2, 0.5f, 0.25f}; + for (int k = 0; k < 4; ++k) { tbl[size_t(k)] = bf(r0[k]); tbl[size_t(4 + k)] = bf(r1[k]); nrm[size_t(k)] = bf(nn[k]); } + float out[4] = {0, 0, 0, 0}; + per_layer_gather(tbl.data(), nrm.data(), V, width, 0, out); + const float want0[4] = {2, 4, 1.5f, 1}; + bool ok0 = true; + for (int k = 0; k < 4; ++k) ok0 = ok0 && out[k] == want0[k]; + std::printf("per-layer gather: row0 -> %.2f %.2f %.2f %.2f\n", out[0], out[1], out[2], out[3]); + check(ok0, "per-layer gather: the projection is row * norm, elementwise (1*2, 2*2, 3*0.5, 4*0.25)"); + per_layer_gather(tbl.data(), nrm.data(), V, width, 1, out); + const float want1[4] = {-2, 1, 0, 2}; + bool ok1 = true; + for (int k = 0; k < 4; ++k) ok1 = ok1 && out[k] == want1[k]; + check(ok1, "per-layer gather: a different row picks a different row of the table (-1*2, 0.5*2, 0*0.5, 8*0.25)"); + // a null norm means "no projection", not "zero": the row passes through + per_layer_gather(tbl.data(), nullptr, V, width, 0, out); + bool ok2 = true; + for (int k = 0; k < 4; ++k) ok2 = ok2 && out[k] == r0[k]; + check(ok2, "per-layer gather: no norm passes the row through (a missing projection is not a zero one)"); + // out-of-range ids and shapes are a no-op, not a crash: the loader can hand a short container + float untouched[4] = {9, 9, 9, 9}; + per_layer_gather(tbl.data(), nrm.data(), V, width, 5, untouched); + per_layer_gather(nullptr, nrm.data(), V, width, 0, untouched); + bool ok3 = true; + for (int k = 0; k < 4; ++k) ok3 = ok3 && untouched[k] == 9; + check(ok3, "per-layer gather: an out-of-range id or a missing table leaves the output untouched"); +} + int main(int argc, char** argv) { const std::string golden_dir = argc > 1 ? argv[1] : ""; const std::string pack = argc > 2 ? argv[2] : ""; @@ -3000,6 +3038,7 @@ int main(int argc, char** argv) { test_fconst_in_component(golden_dir.empty() ? "." : golden_dir); test_embedding_in_component(golden_dir.empty() ? "." : golden_dir); test_per_layer_container(golden_dir.empty() ? "." : golden_dir); + test_per_layer_gather(); test_embedding_f32_component(golden_dir.empty() ? "." : golden_dir); test_qwen3_attn_sections(); test_post_attn_norm_header(golden_dir.empty() ? "." : golden_dir); From b17aea53036fafd4838d19b74a5009e464c2a6d6 Mon Sep 17 00:00:00 2001 From: pi-demo Date: Tue, 10 Nov 2026 00:33:48 -0400 Subject: [PATCH 05/12] npu-pack: dequantise the per-layer CONTEXT projection, and split ok()/complete() (engine#1725, WIP) Tracing the component's graph end to end (not its node list) showed the per-layer input is ( rmsnorm((inputs_embeds @ per_layer_model_projection.weight_t_Q4) * H**-0.5, per_layer_projection_norm) + 16*table[id] ) / sqrt(2) so the table and its norm are only TWO of the three tensors: the context projection is the third, and without it the per-layer input is `16*table/sqrt(2)` -- plausible, and wrong at every layer. The component carries it quantised (`per_layer_model_projection.weight_t_Q4` [8960,48,16], `..._t_scales`, packed `..._t_zero_point`), and the loader wants it dequantised as `model.per_layer_model_proj.weight_layer{l}` [P][H] BF16 per layer. * `read_embedding_per_layer(pack_dir, hidden, why)` now dequantises that projection through the file's own `decode_matmulnbits()` (same decoder the rest of the importer uses) and holds it as [NL][P][H] f32; `hidden` is a parameter because it comes from the pack's geometry, not from the component. * `ok()` is what the reader guarantees (table + norm); `complete()` additionally requires the projection. A component with the first but not the second imports the table and is refused by the LOADER on `per_layer_model_proj.weight_layer0`, which is the honest place to refuse -- the importer does not know whether the decoder declares per_layer_inputs. Measured on `amd/gemma-4-e2b-it-mobius-int4_rai_1.9.0_npu`'s component: table 262144x8960 f32, norm 256 (shared by all 35 layers -- the graph reshapes to [.., 35, 256] around the norm), projection K=1536 N=8960 block 32 int4. WIP: the container still writes the two old names; the rename to the loader's names and the projection emission are the next commit, together with the forward's BF16 band path for the table. --- npu/pack_import.cpp | 53 ++++++++++++++++++++++++++++++++-- npu/pack_import.h | 25 ++++++++++++---- tests/npu_pack_import_test.cpp | 2 +- 3 files changed, 72 insertions(+), 8 deletions(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 7561507b9..83aab41e2 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -917,7 +917,7 @@ DeclaredMap read_declared(const std::string& pack_dir) { // dtype rule slice 1 established: f32 is read, anything else unquantised is refused by dtype rather than mis-read. // Kept separate from read_embedding_component() because the shapes differ (V x width and a width-wide norm, not V x H) // and because the consumer differs: the token table is the head's ORACLE, these are a decoder INPUT. -EmbedPerLayer read_embedding_per_layer(const std::string& pack_dir, std::string* why) { +EmbedPerLayer read_embedding_per_layer(const std::string& pack_dir, int64_t hidden, std::string* why) { EmbedPerLayer out; std::string gpath = pack_dir + "/embedding.onnx"; { @@ -981,6 +981,54 @@ EmbedPerLayer read_embedding_per_layer(const std::string& pack_dir, std::string* !norm->location.empty()) { if (!read_f32(*norm, size_t(width), "per_layer_projection_norm", &out.norm)) return out; } + // The context projection: `per_layer_model_projection.weight_t_Q4` [width][H] at block 32, with `..._t_scales` and + // packed `..._t_zero_point`. Dequantised through the file's own MatMulNBits decoder so the arithmetic is the one the + // rest of the importer uses, then split into one [P][H] matrix per layer for the loader's naming. + { + const OnnxTensor* pw = nullptr; + const OnnxTensor* ps = nullptr; + const OnnxTensor* pz = nullptr; + for (const auto& t : em.tensors) { + if (t.name.find("per_layer_model_projection") == std::string::npos) continue; + if (t.name.find("scale") != std::string::npos) ps = &t; + else if (t.name.find("zero_point") != std::string::npos) pz = &t; + else if (t.name.find("_Q4") != std::string::npos) pw = &t; + } + if (pw && ps && hidden > 0 && width > 0) { + const std::string gdir = fs::path(gpath).parent_path().string(); + const auto blob = [&](const OnnxTensor& t, std::string* w4) -> const uint8_t* { + if (t.location.empty()) return t.raw; + size_t sz = 0; + return map_file(gdir + "/" + t.location, sz, w4); + }; + std::string wa, wb, wc; + const uint8_t* qw = blob(*pw, &wa); + const uint8_t* sc = blob(*ps, &wb); + const uint8_t* zp = pz ? blob(*pz, &wc) : nullptr; + if (qw && sc) { + const int64_t H = hidden; + const Q4 q = decode_matmulnbits(qw, reinterpret_cast(sc), zp, int(width), int(H), 32, + false, zp == nullptr); + if (q.N == int(width) && q.K == int(H)) { + // NL = width / P; the norm is P wide, so P comes from it when the component states one. + const int64_t Pn = !out.norm.empty() ? int64_t(out.norm.size()) : width / (width / 256 ? width / 256 : 1); + out.P = Pn > 0 ? Pn : width; + out.NL = width / out.P; + if (out.NL > 0 && out.NL * out.P == width) { + out.model_proj.assign(size_t(out.NL) * size_t(out.P) * size_t(H), 0.0f); + for (int64_t n = 0; n < width; ++n) + for (int64_t k = 0; k < H; ++k) { + const size_t ci = size_t(n) * size_t(H) + size_t(k); + const size_t gi = size_t(n) * size_t(H / 32) + size_t(k / 32); + out.model_proj[size_t(n) * size_t(H) + size_t(k)] = + float(q.codes[ci]) * q.scale[gi] + q.zero[gi]; + } + out.H = H; + } + } + } + } + } if (out.norm.empty()) { // No projection norm: the table is still usable, but say so — a consumer that expects the projection to be // normalised would otherwise silently use raw rows. @@ -988,6 +1036,7 @@ EmbedPerLayer read_embedding_per_layer(const std::string& pack_dir, std::string* } out.V = V; out.width = width; + if (out.P <= 0) out.P = int64_t(out.norm.size()); // the norm is P wide: it states the per-layer width return out; } @@ -3112,7 +3161,7 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons // than a refusal. { std::string pl_why; - const EmbedPerLayer pl = read_embedding_per_layer(pack_dir, &pl_why); + const EmbedPerLayer pl = read_embedding_per_layer(pack_dir, H, &pl_why); if (pl.ok()) { w.add("model.embed_tokens_per_layer.weight", "BF16", {pl.V, pl.width}, as_bf16(reinterpret_cast(pl.table.data()), kF32, pl.table.size())); diff --git a/npu/pack_import.h b/npu/pack_import.h index 5c819e357..adb77ebf8 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -84,12 +84,27 @@ std::vector read_embedding_component(const std::string& pack_dir, std::st // Read separately from the token table because they have different shapes and different consumers: the token table // becomes the head oracle, this becomes an input. Empty (with `why`) when the component has no per-layer table. struct EmbedPerLayer { - std::vector table; // [V][layers*head_dim], row-major - std::vector norm; // [layers*head_dim] - int64_t V = 0, width = 0; - bool ok() const { return V > 0 && width > 0 && table.size() == size_t(V) * size_t(width) && norm.size() == size_t(width); } + std::vector table; // [V][layers*P], row-major (embed_tokens_per_layer.weight) + std::vector norm; // [P] (per_layer_projection_norm.weight, shared by every layer) + // The CONTEXT projection: the component carries it quantised as + // `per_layer_model_projection.weight_t_Q4` [layers*P][H] plus scales and packed zero points. The loader wants it + // dequantised, one [P][H] matrix per layer (`model.per_layer_model_proj.weight_layer{l}`), so it is dequantised here + // -- without it the per-layer input is `16*table/sqrt(2)`, plausible and wrong at every layer (engine#1725). + std::vector model_proj; // [layers][P][H] + int64_t V = 0, width = 0, P = 0, H = 0, NL = 0; + // `ok()` is what the reader guarantees: the table and the projection norm. `complete()` additionally requires the + // context projection, which the PLE contract needs -- a component with `ok()` but not `complete()` imports the table + // and will be refused by the loader on `per_layer_model_proj.weight_layer0`, which is the honest place to refuse + // (the importer does not know whether the decoder declares per_layer_inputs). + bool ok() const { + return V > 0 && width > 0 && table.size() == size_t(V) * size_t(width) && + (!norm.empty() ? norm.size() == size_t(P) : true); + } + bool complete() const { return ok() && P > 0 && H > 0 && NL > 0 && model_proj.size() == size_t(NL) * size_t(P) * size_t(H); } }; -EmbedPerLayer read_embedding_per_layer(const std::string& pack_dir, std::string* why); +// `hidden` is the decoder's hidden size (the context projection's K); it comes from the pack's geometry, not from +// the component, which is why the caller passes it. +EmbedPerLayer read_embedding_per_layer(const std::string& pack_dir, int64_t hidden, std::string* why); // One tensor the pack describes, and where its values would come from. struct Planned { diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 96c53ab2c..d17532adc 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -2945,7 +2945,7 @@ void test_per_layer_container(const std::string& root) { R"({"model_type":"gemma4","hidden_size":3,"intermediate_size":6,"num_hidden_layers":1,)" R"("num_attention_heads":1,"num_key_value_heads":1,"head_dim":3,"vocab_size":4,"rope_theta":10000.0})"); std::string why; - const pi::EmbedPerLayer pl = pi::read_embedding_per_layer(dir.string(), &why); + const pi::EmbedPerLayer pl = pi::read_embedding_per_layer(dir.string(), 3, &why); std::printf("per-layer: V %lld width %lld, why: %s (tok %zu per %zu norm %zu bytes)\n", (long long) pl.V, (long long) pl.width, why.c_str(), tok_bytes, per_bytes, norm.size() * 4); check(pl.ok(), "per-layer: the table and its projection norm are read from the component genai_config.json names"); From 83c323ba088975cd3959954e0294266d0f2306e8 Mon Sep 17 00:00:00 2001 From: pi-demo Date: Tue, 10 Nov 2026 00:51:59 -0400 Subject: [PATCH 06/12] npu-pack: write the per-layer PLE tensors under the names the forward loads (engine#1725, WIP) The forward's `build_ple_inputs()`/`load_ple_block()` already implement this family's arithmetic -- I traced the component's graph and it is line for line the same: `context = rmsnorm((emb @ W_l) * H**-0.5, norm)`, `identity = 16 * table[id]`, `per_layer_input = (context + identity)/sqrt(2)` -- but it reads different names than the importer wrote. This commit closes that gap: model.per_layer_token_embd.weight BF16 {V, NL*P} (was model.embed_tokens_per_layer.weight) model.per_layer_proj_norm.weight BF16 {P} (was model.per_layer_projection_norm.weight) model.per_layer_model_proj.weight_layer{l} BF16 {P, H}, l = 0..NL-1 (was absent) The token table is **streamed**, never materialised: `read_per_layer_table_band()` maps the component's external data and converts 32 rows at a time from f32 to BF16 straight into the container blob, because the full table is V x NL*P f32 = 9.4 GiB on gemma-4-e2b and a converting copy of that would be the largest allocation in the import. Two bugs of my own found while writing it, both worth the comments they carry: * the first version of the band reader did `memcpy(dst, src, n * 4)` into a `uint16_t*` of n elements -- 4 bytes per element into a 2-byte-per-element buffer, i.e. a 2x overflow. Removed before it ran; * the projection matrices were being dequantised but never written; `pl.complete()` (table + norm + projection) is what now gates the per-layer emission, and a component with `ok()` but not `complete()` still writes the table, leaving the refusal to the LOADER (`missing model.per_layer_model_proj.weight_layer0`), which is the honest place. All existing checks pass; copyright rc 0. WIP: the forward still lacks a RAW-BF16 band path in `load_ple_block` (it handles plain int8 + scale, or the chunked container formats), so this container is one loader branch away from being readable. --- npu/pack_import.cpp | 109 ++++++++++++++++++++++++++++++++++++++++---- npu/pack_import.h | 6 +++ 2 files changed, 105 insertions(+), 10 deletions(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 83aab41e2..86c2ef97a 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -1040,6 +1040,65 @@ EmbedPerLayer read_embedding_per_layer(const std::string& pack_dir, int64_t hidd return out; } +// engine#1725: stream the per-layer token table out in bands. The component states it as f32 (V x NL*P, 9.4 GiB on +// gemma-4-e2b), so materialising it to convert would cost more RAM than the box has to spare for one import; the bands go +// straight from the mapped external data into the container's BF16 layout. +int64_t read_per_layer_table_band(const std::string& pack_dir, EmbedPerLayer* meta, int64_t first_row, int64_t rows, + uint8_t* out, std::string* why) { + if (meta == nullptr || out == nullptr || meta->V <= 0 || meta->width <= 0) { + if (why) *why = "no per-layer table metadata (call read_embedding_per_layer first)"; + return -1; + } + if (first_row < 0 || rows <= 0 || first_row + rows > meta->V) { + if (why) *why = "band out of range"; + return -1; + } + std::string gpath = pack_dir + "/embedding.onnx"; + { + std::ifstream gf(pack_dir + "/genai_config.json"); + if (gf) { + json g; + try { g = json::parse(gf); } catch (...) { g = json::object(); } + if (g.is_object() && g.contains("model") && g["model"].is_object() && g["model"].contains("embedding") && + g["model"]["embedding"].is_object() && g["model"]["embedding"].contains("filename") && + g["model"]["embedding"]["filename"].is_string()) { + const std::string rel = g["model"]["embedding"]["filename"].get(); + if (!rel.empty() && rel.front() != '/' && rel.find("..") == std::string::npos) { + std::error_code nec; + if (fs::is_regular_file(pack_dir + "/" + rel, nec)) gpath = pack_dir + "/" + rel; + } + } + } + } + OnnxModel em; + try { em = parse_onnx(gpath); } catch (const std::exception& e) { if (why) *why = e.what(); return -1; } + const OnnxTensor* table = nullptr; + for (const auto& t : em.tensors) + if (t.name.find("embed_tokens_per_layer") != std::string::npos) table = &t; + if (table == nullptr) { if (why) *why = "the component carries no embed_tokens_per_layer table"; return -1; } + if (table->location.empty()) { if (why) *why = "the per-layer table has no external data (banded reads need it)"; return -1; } + const std::string base = fs::path(gpath).parent_path().string() + "/" + table->location; + size_t sz = 0; + std::string w2; + const uint8_t* m = map_file(base, sz, &w2); + if (!m) { if (why) *why = w2; return -1; } + const uint64_t row_bytes = uint64_t(meta->width) * 4ull; + const uint64_t need_end = table->offset + uint64_t(first_row + rows) * row_bytes; + if (need_end > sz) { if (why) *why = "the per-layer table declares more bytes than its data carries"; return -1; } + const uint8_t* src = m + table->offset + uint64_t(first_row) * row_bytes; + uint16_t* dst = reinterpret_cast(out); + const size_t n = size_t(rows) * size_t(meta->width); + for (size_t i = 0; i < n; ++i) { + float f; + std::memcpy(&f, src + i * 4, 4); + uint32_t b; + std::memcpy(&b, &f, 4); + const uint32_t lsb = (b >> 16) & 1u; + dst[i] = uint16_t((b >> 16) + (lsb && ((b & 0xFFFFu) != 0 || (((b >> 16) & 1u) != 0)) ? 1u : 0u)); + } + return rows; +} + std::vector read_embedding_component(const std::string& pack_dir, std::string* why) { // A multi-component pack (`type: compositemodel`) names its embedding component in genai_config.json // (`model.embedding.filename = "embedding/model.onnx"`), exactly as it names its decoder; a single-graph pack keeps @@ -3155,19 +3214,49 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons embed_tbl = EmbedTable{embed_bf16.data(), EmbedTable::Fmt::BF16, int(H)}; w.add("model.embed_tokens.weight", "BF16", {V, H}, embed_bf16); ++r.tensors; - // engine#1725: a multi-component pack's embedding also carries the PER-LAYER table and its projection norm, which - // the decoder consumes as `per_layer_inputs`. Written as BF16 like every other table here (the component ships - // f32); the forward path does the gather. Absent for a single-graph pack, which is why this is conditional rather - // than a refusal. + // engine#1725: a multi-component pack's embedding also carries the PER-LAYER table, its projection norm and the + // context projection. Written under the names the FORWARD loads (q4nx_forward.cpp's build_ple_inputs/load_ple_block), + // because the arithmetic there is already this model's: context = rmsnorm((emb @ W_l) * H**-0.5, norm), + // identity = 16 * table[id], per_layer_input = (context + identity)/sqrt(2). Absent for a single-graph pack. { std::string pl_why; - const EmbedPerLayer pl = read_embedding_per_layer(pack_dir, H, &pl_why); + EmbedPerLayer pl = read_embedding_per_layer(pack_dir, H, &pl_why); if (pl.ok()) { - w.add("model.embed_tokens_per_layer.weight", "BF16", {pl.V, pl.width}, - as_bf16(reinterpret_cast(pl.table.data()), kF32, pl.table.size())); - w.add("model.per_layer_projection_norm.weight", "BF16", {pl.width}, - as_bf16(reinterpret_cast(pl.norm.data()), kF32, pl.norm.size())); - r.tensors += 2; + // The token table is streamed: V x (NL*P) f32 is 9.4 GiB on e2b, and a band is 32 rows. + std::vector tbl_bf16(size_t(pl.V) * size_t(pl.width) * 2); + bool bands_ok = true; + for (int64_t row = 0; row < pl.V; row += 32) { + const int64_t n = (pl.V - row < 32) ? (pl.V - row) : 32; + if (read_per_layer_table_band(pack_dir, &pl, row, n, + tbl_bf16.data() + size_t(row) * size_t(pl.width) * 2, &pl_why) != n) { + bands_ok = false; + break; + } + } + if (bands_ok) { + w.add("model.per_layer_token_embd.weight", "BF16", {pl.V, pl.width}, std::move(tbl_bf16)); + if (!pl.norm.empty()) + w.add("model.per_layer_proj_norm.weight", "BF16", {pl.P}, + as_bf16(reinterpret_cast(pl.norm.data()), kF32, pl.norm.size())); + r.tensors += 1 + (pl.norm.empty() ? 0 : 1); + if (pl.complete()) { + for (int64_t l = 0; l < pl.NL; ++l) { + std::vector one(size_t(pl.P) * size_t(pl.H) * 2); + for (int64_t o = 0; o < pl.P; ++o) + for (int64_t k = 0; k < pl.H; ++k) { + const float f = pl.model_proj[size_t(l * pl.P + o) * size_t(pl.H) + size_t(k)]; + uint32_t b; + std::memcpy(&b, &f, 4); + const uint32_t lsb = (b >> 16) & 1u; + reinterpret_cast(one.data())[size_t(o) * size_t(pl.H) + size_t(k)] = + uint16_t((b >> 16) + (lsb && ((b & 0xFFFFu) != 0 || (((b >> 16) & 1u) != 0)) ? 1u : 0u)); + } + w.add("model.per_layer_model_proj.weight_layer" + std::to_string(l), "BF16", {pl.P, pl.H}, + std::move(one)); + ++r.tensors; + } + } + } } } } diff --git a/npu/pack_import.h b/npu/pack_import.h index adb77ebf8..2b04aad3d 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -106,6 +106,12 @@ struct EmbedPerLayer { // the component, which is why the caller passes it. EmbedPerLayer read_embedding_per_layer(const std::string& pack_dir, int64_t hidden, std::string* why); +// The per-layer token table is `V x (NL*P)` f32 in the component -- 9.4 GiB for gemma-4-e2b -- so it is never +// materialised: the importer streams it into the container in 32-row bands, converting to BF16 as it goes (engine#1725). +// Returns the rows written, or -1 with `why` set. `out` must hold `rows * width * 2` bytes. +int64_t read_per_layer_table_band(const std::string& pack_dir, EmbedPerLayer* meta, int64_t first_row, int64_t rows, + uint8_t* out, std::string* why); + // One tensor the pack describes, and where its values would come from. struct Planned { std::string name; // Q4NX tensor name (model.layers.N.<...>.weight) From fbe886171f9250054de26f110f56cdd74b428330 Mon Sep 17 00:00:00 2001 From: pi-demo Date: Tue, 10 Nov 2026 01:23:13 -0400 Subject: [PATCH 07/12] forward: read the per-layer token table as raw BF16 bands (engine#1725) `load_ple_block()` handled two shapes: plain int8 + a per-32-column `.scale`, and the chunked container formats. The multi-component packs state this table as f32 and the importer converts it to BF16 when it writes the container -- a converting copy of the whole V x NL*P table would be 9.4 GiB on gemma-4-e2b -- so the container's table is raw BF16 and neither branch matched it. The band is the same shape the int8 branch handles (32 rows x cols), with a 2-byte element instead of a code, so this is a loop of `bf16_to_f32` over the band, chosen by data_size: a multiple of the BF16 band and NOT the plain-int8 size. That ordering matters -- `plain_bytes` (one byte per element) and a BF16 band differ by a factor of two, and testing int8 first would take a BF16 table whose size happens to equal `plain_bytes` for another geometry. With this, a container written by the importer (#1725's `83c323b`) is readable end to end on the embedding side: table (banded), norm (256), and the per-layer context projections. --- npu/forward/src/q4nx_forward.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 444356ee1..07e6ff160 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -2683,7 +2683,15 @@ bool Q4nxNpuForward::load_ple_block(int row) { ple_block_.assign((size_t)Q4NX_TILE_ROWS * (size_t)cols, 0.0f); const size_t plain_bytes = (size_t)sem_.vocab_size_per_layer_input * (size_t)cols; - if ((size_t)td->data_size == plain_bytes) { + // engine#1725: a RAW BF16 band. The multi-component packs state this table as f32 and the importer converts it to BF16 + // when it writes the container (a converting copy of the whole V x NL*P table would be 9.4 GiB), so the band here is + // 32 rows of BF16 -- exactly the shape the plain-int8 branch handles, with a 2-byte element instead of a code. + const size_t bf16_bytes = (size_t)Q4NX_TILE_ROWS * (size_t)cols * 2; + if ((size_t)td->data_size >= ((size_t)block + 1) * bf16_bytes && + (size_t)td->data_size % bf16_bytes == 0 && (size_t)td->data_size != plain_bytes) { + const uint16_t* band = reinterpret_cast(raw + (size_t)block * bf16_bytes); + for (size_t i = 0; i < (size_t)Q4NX_TILE_ROWS * (size_t)cols; ++i) ple_block_[i] = bf16_to_f32(band[i]); + } else if ((size_t)td->data_size == plain_bytes) { // PLAIN int8 + per-32-column scale (Gemma4): only the requested band of // 32 rows is needed, so it is dequantized in place rather than the whole // [vocab, NL*ple_dim] table. From 6514cb154afebfe95e69b87abc9b8946e7de5368 Mon Sep 17 00:00:00 2001 From: pi-demo Date: Tue, 10 Nov 2026 02:31:27 -0400 Subject: [PATCH 08/12] npu-pack: retire the table-times-norm primitive -- it is not this model's arithmetic (engine#1725) The previous commit on this branch added `per_layer_gather()` computing `table[id] * norm`, from a reading of the component's node LIST. Tracing the component's graph end to end showed something else: per_layer_inputs = ( rmsnorm((inputs_embeds @ per_layer_model_projection.weight_t_Q4) * H**-0.5, per_layer_projection_norm) + 16*table[id] ) / sqrt(2) -- the norm applies to the projected CONTEXT, the table row is ADDED (times 16 = sqrt(256)), and the sum is scaled by 1/sqrt(2). That is exactly what the forward's `build_ple_inputs()` already computes, so the primitive was not merely unused, it was wrong, and a pure function with a passing test is exactly the wrong thing to leave in a tree for someone to call later. Deleted: `npu/forward/include/per_layer_embed.h`, its test, and the test target's include line it needed. The container and loader work that DOES match the graph stays (earlier commits on this branch): the three PLE tensors written under the names the forward reads, the streamed BF16 table, and `load_ple_block()`'s raw-BF16 band path. Deleting beats keeping a wrong primitive as documentation: the retraction is recorded on engine#1725 with the graph, which is where someone changing this will look. --- CMakeLists.txt | 1 - npu/forward/include/per_layer_embed.h | 50 --------------------------- tests/npu_pack_import_test.cpp | 38 -------------------- 3 files changed, 89 deletions(-) delete mode 100644 npu/forward/include/per_layer_embed.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 9e1bc5fe9..78503673a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1061,7 +1061,6 @@ target_include_directories(npu_q4nx_test PRIVATE app) target_link_libraries(npu_q4nx_test PRIVATE onebit_npu) add_test(NAME npu_q4nx COMMAND npu_q4nx_test ${CMAKE_SOURCE_DIR}/tests/golden/q4nx) add_executable(npu_pack_import_test tests/npu_pack_import_test.cpp) -target_include_directories(npu_pack_import_test PRIVATE ${CMAKE_SOURCE_DIR}/npu/forward/include) target_link_libraries(npu_pack_import_test PRIVATE onebit_npu) add_test(NAME npu_pack_import COMMAND npu_pack_import_test ${CMAKE_SOURCE_DIR}/tests/golden) # engine#1726: NPU_INFER_DUMP_DIR's residual write must SAY when it cannot write -- a missing or unwritable directory, or a diff --git a/npu/forward/include/per_layer_embed.h b/npu/forward/include/per_layer_embed.h deleted file mode 100644 index db86af573..000000000 --- a/npu/forward/include/per_layer_embed.h +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2026 bong-water-water-bong -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// The per-layer embedding a multi-component decoder consumes as `per_layer_inputs` (engine#1725). -// -// gemma-4 1.9 packs ship it beside the token table: `embed_tokens_per_layer.weight` (V x layers*head_dim) and -// `per_layer_projection_norm.weight` (layers*head_dim), both f32 in the pack and BF16 in our container. A decoder run -// gathers ONE ROW for the token, then projects it through the 256-wide (per layer) norm -- that value is the layer's -// per-layer input. Pure, header-only, no container and no device, so the projection can be pinned by a test with three -// numbers in it. -#pragma once - -#include -#include - -namespace onebit::npu { - -// bf16 bits -> float, round-to-nearest-even on the widened mantissa (the bits are already bf16, so this is exact). -inline float pl_bf16_to_f32(uint16_t v) { - const uint32_t bits = uint32_t(v) << 16; - float f; - __builtin_memcpy(&f, &bits, 4); - return f; -} - -// out[0..width) = table[id][k] * norm[k] for k in [0, width). Sub-normal and zero norms are passed through as a plain -// multiply (a zero norm zeroes that entry, which is what the arithmetic says; no special case, no clamp). -inline void per_layer_gather(const uint16_t* table, const uint16_t* norm, int64_t V, int64_t width, int64_t id, - float* out) { - if (table == nullptr || out == nullptr || V <= 0 || width <= 0 || id < 0 || id >= V) return; - const uint16_t* row = table + size_t(id) * size_t(width); - for (int64_t k = 0; k < width; ++k) { - const float w = pl_bf16_to_f32(row[size_t(k)]); - out[k] = norm != nullptr ? w * pl_bf16_to_f32(norm[size_t(k)]) : w; - } -} - -} // namespace onebit::npu diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index d17532adc..1c8983018 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -37,7 +37,6 @@ #include "pack.h" #include "pack_import.h" #include "q4nx.h" -#include "per_layer_embed.h" #include "q4nx_device.h" #include @@ -2961,42 +2960,6 @@ void test_per_layer_container(const std::string& root) { } -// engine#1725: the per-layer gather the forward will call. Pure: three numbers pin the projection. -void test_per_layer_gather() { - using namespace onebit::npu; - const int64_t V = 2, width = 4; - // two-arg form: `nrm(size_t(width))` would be a function declaration (the same trap as the per-layer fixture above) - std::vector tbl(size_t(V * width), 0), nrm(size_t(width), 0); - auto bf = [](float f) { uint32_t b; __builtin_memcpy(&b, &f, 4); return uint16_t(b >> 16); }; - // row 0 = {1, 2, 3, 4}, row 1 = {-1, 0.5, 0, 8}; norm = {2, 2, 0.5, 0.25} - const float r0[4] = {1, 2, 3, 4}, r1[4] = {-1, 0.5f, 0, 8}, nn[4] = {2, 2, 0.5f, 0.25f}; - for (int k = 0; k < 4; ++k) { tbl[size_t(k)] = bf(r0[k]); tbl[size_t(4 + k)] = bf(r1[k]); nrm[size_t(k)] = bf(nn[k]); } - float out[4] = {0, 0, 0, 0}; - per_layer_gather(tbl.data(), nrm.data(), V, width, 0, out); - const float want0[4] = {2, 4, 1.5f, 1}; - bool ok0 = true; - for (int k = 0; k < 4; ++k) ok0 = ok0 && out[k] == want0[k]; - std::printf("per-layer gather: row0 -> %.2f %.2f %.2f %.2f\n", out[0], out[1], out[2], out[3]); - check(ok0, "per-layer gather: the projection is row * norm, elementwise (1*2, 2*2, 3*0.5, 4*0.25)"); - per_layer_gather(tbl.data(), nrm.data(), V, width, 1, out); - const float want1[4] = {-2, 1, 0, 2}; - bool ok1 = true; - for (int k = 0; k < 4; ++k) ok1 = ok1 && out[k] == want1[k]; - check(ok1, "per-layer gather: a different row picks a different row of the table (-1*2, 0.5*2, 0*0.5, 8*0.25)"); - // a null norm means "no projection", not "zero": the row passes through - per_layer_gather(tbl.data(), nullptr, V, width, 0, out); - bool ok2 = true; - for (int k = 0; k < 4; ++k) ok2 = ok2 && out[k] == r0[k]; - check(ok2, "per-layer gather: no norm passes the row through (a missing projection is not a zero one)"); - // out-of-range ids and shapes are a no-op, not a crash: the loader can hand a short container - float untouched[4] = {9, 9, 9, 9}; - per_layer_gather(tbl.data(), nrm.data(), V, width, 5, untouched); - per_layer_gather(nullptr, nrm.data(), V, width, 0, untouched); - bool ok3 = true; - for (int k = 0; k < 4; ++k) ok3 = ok3 && untouched[k] == 9; - check(ok3, "per-layer gather: an out-of-range id or a missing table leaves the output untouched"); -} - int main(int argc, char** argv) { const std::string golden_dir = argc > 1 ? argv[1] : ""; const std::string pack = argc > 2 ? argv[2] : ""; @@ -3038,7 +3001,6 @@ int main(int argc, char** argv) { test_fconst_in_component(golden_dir.empty() ? "." : golden_dir); test_embedding_in_component(golden_dir.empty() ? "." : golden_dir); test_per_layer_container(golden_dir.empty() ? "." : golden_dir); - test_per_layer_gather(); test_embedding_f32_component(golden_dir.empty() ? "." : golden_dir); test_qwen3_attn_sections(); test_post_attn_norm_header(golden_dir.empty() ? "." : golden_dir); From f026d686885d3263ef93c327d8fe1fd0cc9207c4 Mon Sep 17 00:00:00 2001 From: pi-demo Date: Tue, 10 Nov 2026 03:36:29 -0400 Subject: [PATCH 09/12] npu-pack: tell a variable-stride 1.9 fconst from "not a 1.9 fconst" (engine#1503) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `fconst19_detect()` requires ONE stride for the whole file. The mobius line does not have one -- measured on gemma-4-e2b: 35 blocks, each opening with 8 anchors (8 heads x 4 B = 32-byte header), sizes 21.87 MiB to 288.06 MiB -- so the detect fails and the pack was reported the same way as a file that is simply not a 1.9 fconst at all. * `fconst19_block_starts()` is the run scan on its own (exactly `layers` runs of `heads*4` anchor bytes), so a caller can tell the two cases apart before the stride test; * when the runs are there but the strides differ, the refusal now NAMES the structure: how many blocks, where the first one starts, and the size range in KiB, plus why the rule table cannot be the answer ("one stride per file" is the table's model) and what the answer is (the per-layer block table; the anchors give the starts, and each block's size follows that layer's own projection shapes). On the real pack: 1bit npu-pack: … this IS a Ryzen AI 1.9 Token .fconst (35 blocks of 32 B headers at 2564864 B) but its blocks are NOT uniform: 22395 KiB to 294973 KiB, i.e. a variable-stride container. The rule table describes one stride per file, so no row can fit it; this family (mobius/MatFormer) needs the per-layer block table instead … Crafted arm: four blocks with different body sizes are refused by name (`variable stride: refused by name (variable-stride container)`), so the distinction is pinned without a 1.3 GiB pack. --- npu/pack_import.cpp | 44 ++++++++++++++++++++++++++++++++ tests/npu_pack_import_test.cpp | 46 ++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 86c2ef97a..7e9624639 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -485,6 +485,27 @@ std::vector fconst19_header(int heads) { // True when `fc` is a 1.9 Token .fconst of this geometry: exactly `layers` headers, evenly spaced. // Fills the first header's offset and the block stride. False means the 1.8 form (or a shape this // importer does not know, which the caller decides about). +// engine#1503: the run scan on its own. `fconst19_detect` additionally requires ONE stride, which the mobius line +// (gemma-4-e2b: 35 blocks of 21.87-288.06 MiB) does not have -- so a caller that needs to tell "this is not a 1.9 fconst" +// from "this is a 1.9 fconst whose blocks are not uniform" needs the starts before the stride test. +bool fconst19_block_starts(const Fconst& fc, int heads, int layers, std::vector* starts) { + if (heads <= 0 || layers < 2) return false; + const std::vector pat = fconst19_header(heads); + std::vector runs; + for (size_t i = 0; i + pat.size() <= fc.size;) { + if (fc.map[i] == pat[0] && fc.map[i + 2] == 0x7f && fc.map[i + 3] == 0xff && + std::memcmp(fc.map + i, pat.data(), pat.size()) == 0) { + runs.push_back(int64_t(i)); + i += pat.size(); + } else { + ++i; + } + } + if (int(runs.size()) != layers) return false; + if (starts) *starts = std::move(runs); + return true; +} + bool fconst19_detect(const Fconst& fc, int heads, int layers, int64_t* hdr_off, int64_t* stride) { if (heads <= 0 || layers < 2) return false; const std::vector pat = fconst19_header(heads); @@ -519,6 +540,29 @@ std::string canonical_token_fconst(const std::string& path, int hidden, int head Fconst fc = open_fconst(path); int64_t hdr_off = 0, stride9 = 0; if (!fconst19_detect(fc, heads, layers, &hdr_off, &stride9)) { + // Distinguish "not a 1.9 fconst" from "a 1.9 fconst whose blocks are NOT uniform": the mobius line (gemma-4-e2b + // measured: 35 blocks, 8 anchors each, sizes 21.87-288.06 MiB) is the second, and the rule table cannot describe + // it -- one stride per file is the table's model (engine#1503). Refusing with the block table is the difference + // between a reader that looks unmeasured and a model that is wrong for the family. + std::vector starts; + if (fconst19_block_starts(fc, heads, layers, &starts)) { + std::string sizes; + int64_t lo = INT64_MAX, hi = 0; + for (size_t k = 0; k < starts.size(); ++k) { + const int64_t end = (k + 1 < starts.size()) ? starts[k + 1] : int64_t(fc.size); + const int64_t sz = end - starts[k]; + if (sz < lo) lo = sz; + if (sz > hi) hi = sz; + } + unmap_file(fc.map, fc.size); + throw std::runtime_error( + path + ": this IS a Ryzen AI 1.9 Token .fconst (" + std::to_string(layers) + " blocks of " + + std::to_string(heads * 4) + " B headers at " + std::to_string(starts.front()) + " B) but its blocks are " + + "NOT uniform: " + std::to_string(lo / 1024) + " KiB to " + std::to_string(hi / 1024) + + " KiB, i.e. a variable-stride container. The rule table describes one stride per file, so no row can fit " + "it; this family (mobius/MatFormer) needs the per-layer block table instead -- the anchors give the block " + "starts, and each block's size follows that layer's own projection shapes (engine#1503)"); + } unmap_file(fc.map, fc.size); return path; } diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 1c8983018..3bba7447b 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -2960,6 +2960,51 @@ void test_per_layer_container(const std::string& root) { } + +// engine#1503: a 1.9 fconst whose blocks are NOT uniform (the mobius line). `fconst19_detect` requires one stride for the +// whole file, so it fails -- and the failure used to be indistinguishable from "not a 1.9 fconst at all". The refusal now +// names the structure and says the rule table cannot describe it. Measured on gemma-4-e2b: 35 blocks of 8 anchors, sizes +// 21.87-288.06 MiB. +void test_fconst_variable_stride(const std::string& root) { + const fs::path dir = fs::path(root) / "fconst-varstride"; + fs::remove_all(dir); + fs::create_directories(dir / "cache"); + const int H = 128, L = 4, heads = 2; + const int64_t hdr = heads * 4; + // four blocks, each header then a body of a DIFFERENT size: 64 KiB, 96 KiB, 128 KiB, 160 KiB + const int64_t bodies[4] = {64 << 10, 96 << 10, 128 << 10, 160 << 10}; + std::string fc; + for (int l = 0; l < L; ++l) { + for (int h = 0; h < heads; ++h) { // 00 00 7f ff per head + fc.push_back('\0'); fc.push_back('\0'); fc.push_back(char(0x7f)); fc.push_back(char(0xff)); + } + fc.append(size_t(bodies[l]), '\0'); + } + check(fc.size() > size_t(L * hdr), "variable stride: the crafted fconst is larger than its headers"); + put_file((dir / "dd_metastate_Llm_Token_Token_crafted.fconst").string(), fc); + put_file((dir / "config.json").string(), + R"({"model_type":"gemma4","hidden_size":128,"intermediate_size":256,"num_hidden_layers":4,)" + R"("num_attention_heads":2,"num_key_value_heads":1,"head_dim":64,"vocab_size":64,"rope_theta":10000.0})"); + put_file((dir / "model.onnx").string(), pb::model({}, {})); + put_file((dir / "model.onnx.data").string(), std::string(4096, '\x11')); + pi::Options allow; + allow.allow_incomplete = true; + const std::string out = (fs::temp_directory_path() / ("varstride_out." + std::to_string(::getpid()))).string(); + fs::remove_all(out); + bool refused = false; + std::string what; + try { + (void) pi::import_pack(dir.string(), out, allow); + } catch (const std::exception& e) { + what = e.what(); + refused = what.find("variable-stride") != std::string::npos || what.find("NOT uniform") != std::string::npos; + } + std::printf("variable stride: %s\n", refused ? "refused by name (variable-stride container)" : what.substr(0, 140).c_str()); + check(refused, "variable stride: a 1.9 fconst whose blocks differ in size is refused BY NAME, not as 'not a 1.9 fconst'"); + fs::remove_all(out); + fs::remove_all(dir); +} + int main(int argc, char** argv) { const std::string golden_dir = argc > 1 ? argv[1] : ""; const std::string pack = argc > 2 ? argv[2] : ""; @@ -2989,6 +3034,7 @@ int main(int argc, char** argv) { test_attn_tile(golden_dir); test_mlp_gu(golden_dir); test_fconst_eps(golden_dir.empty() ? "." : golden_dir); + test_fconst_variable_stride(golden_dir.empty() ? "." : golden_dir); test_attn_payload_off(); test_text_config_geometry(golden_dir.empty() ? "." : golden_dir); test_hybrid_pack_import(golden_dir.empty() ? "." : golden_dir); From 3814c1d71ee629233adb95268c7397873bcfc99f Mon Sep 17 00:00:00 2001 From: pi-demo Date: Tue, 10 Nov 2026 06:53:46 -0400 Subject: [PATCH 10/12] forward: decide the per-layer table's layout by its DECLARED dtype, not by arithmetic on the size Review of #1825 caught a real collision class in my BF16-band branch: it separated the two layouts with data_size >= (block+1)*bf16_bytes && data_size % bf16_bytes == 0 && data_size != plain_bytes and every multiple of the band satisfies the first two, so a plain table whose byte count happens to divide would be read as 32-row BF16 bands -- "plausible and wrong", the failure my own comment warns about. The tree already decides this way elsewhere (`model.c` does `strcmp(dtype, "I8")` on the same field) and `q4nx_dequant.h`'s rule is that an ambiguous tile count is refused rather than resolved by which test happens to pass. * BF16 is taken only when the container SAYS BF16, and then its size must be whole 32-row bands (else refused); * plain I8 (or Q4) keeps the existing int8 + .scale path; * any other DECLARED dtype is refused by name, and so is an EMPTY one -- a container written by this importer always states a dtype (`w.add()` writes it for every tensor), so "no dtype" means a foreign or truncated container, not a layout this loader may guess. `model.c` populates `dtype` for every tensor entry that carries the key, which is what makes the branch sound rather than merely tidier; that is stated in the comment next to the test. --- npu/forward/src/q4nx_forward.cpp | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 07e6ff160..8354f76ae 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -2685,10 +2685,33 @@ bool Q4nxNpuForward::load_ple_block(int row) { (size_t)sem_.vocab_size_per_layer_input * (size_t)cols; // engine#1725: a RAW BF16 band. The multi-component packs state this table as f32 and the importer converts it to BF16 // when it writes the container (a converting copy of the whole V x NL*P table would be 9.4 GiB), so the band here is - // 32 rows of BF16 -- exactly the shape the plain-int8 branch handles, with a 2-byte element instead of a code. + // 32 rows of BF16 -- the shape the plain-int8 branch handles, with a 2-byte element instead of a code. + // + // The branch is chosen by the container's DECLARED dtype, not by arithmetic on data_size (review of #1825): a size test + // needs `% bf16_bytes == 0 && != plain_bytes` to separate two layouts, and every multiple of the band satisfies it, so a + // plain table whose byte count happens to divide would be read as BF16 bands -- plausible and wrong. This tree already + // decides exactly this way elsewhere (`model.c`: `strcmp(dtype,"I8")`), and `q4nx_dequant.h`'s rule is that an + // ambiguous layout is refused rather than resolved by which test happens to pass. const size_t bf16_bytes = (size_t)Q4NX_TILE_ROWS * (size_t)cols * 2; - if ((size_t)td->data_size >= ((size_t)block + 1) * bf16_bytes && - (size_t)td->data_size % bf16_bytes == 0 && (size_t)td->data_size != plain_bytes) { + const char* pl_dtype = td->dtype; + const bool declared_bf16 = pl_dtype[0] != '\0' && std::strcmp(pl_dtype, "BF16") == 0; + const bool declared_i8 = pl_dtype[0] != '\0' && (std::strcmp(pl_dtype, "I8") == 0 || std::strcmp(pl_dtype, "Q4") == 0); + if (!declared_bf16 && !declared_i8) { + // Some other declared format, or none at all: refuse rather than fall through to a layout guess. A container + // written by this importer always states one (`w.add()` writes dtype for every tensor), so an empty dtype means a + // foreign or truncated container, not a layout this loader can infer. + err_ = pl_dtype[0] != '\0' + ? std::string("per_layer_token_embd has dtype ") + pl_dtype + + ", which this loader does not read (BF16 or plain I8 only)" + : std::string("per_layer_token_embd declares no dtype, so its layout cannot be decided"); + return false; + } + if (declared_bf16) { + if ((size_t)td->data_size < ((size_t)block + 1) * bf16_bytes || + (size_t)td->data_size % bf16_bytes != 0) { + err_ = "per_layer_token_embd is declared BF16 but its size is not whole 32-row bands"; + return false; + } const uint16_t* band = reinterpret_cast(raw + (size_t)block * bf16_bytes); for (size_t i = 0; i < (size_t)Q4NX_TILE_ROWS * (size_t)cols; ++i) ple_block_[i] = bf16_to_f32(band[i]); } else if ((size_t)td->data_size == plain_bytes) { From 111ba732508c2555572043bc40c8d48d8590a225 Mon Sep 17 00:00:00 2001 From: pi-demo Date: Tue, 10 Nov 2026 07:17:19 -0400 Subject: [PATCH 11/12] npu-pack: a container that carries the PLE tensors now also says how to switch the block on (engine#1725) The per-layer tensors are useless without the two numbers the forward's semantics reader gates the whole block on (`hidden_size_per_layer_input > 0`, q4nx_semantics.c). gemma-4 states them in model_config.json's text_config (256 and 262144); the container's own config.json did not carry them, so an import would write the table, the norm and the per-layer projections and then the forward would skip the block silently -- the worst failure shape, because the container looks complete. * when the component is `complete()` (table + norm + context projection), the container's config.json gains `hidden_size_per_layer_input` = P and `vocab_size_per_layer_input` = V, taken from the COMPONENT (where they are measured) rather than from the pack's config, which may not state them at all; * the three Gemma-4 layer-shape switches the same reader consults -- `num_kv_shared_layers`, `use_double_wide_mlp`, `extra_layer_norms` -- are carried through when model_config.json's text_config states them, and left absent otherwise so the reader's defaults apply. The read happens at the config-writing site, which lives in a different writer from the one that emits the tensors: one metadata parse (names, dims, the quantised projection header -- the 9.4 GiB table is NOT read there) rather than plumbing a value across two functions. --- npu/pack_import.cpp | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 7e9624639..3113f320b 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -4161,6 +4161,38 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons c["vocab_size"] = p.dims.vocab; c["model_type"] = p.dims.model_type; c["rope_theta"] = p.dims.rope_theta; + // engine#1725: the PLE block is switched ON by `hidden_size_per_layer_input > 0` in the container's semantics + // (q4nx_semantics.c), so a container that carries the per-layer tensors but not these two keys would import them + // and never use them -- the forward skips the block silently. Taken from the component itself, which is where they + // are measured (gemma-4-e2b: 256 and 262144), rather than from the pack's config, which states them one level down. + { + // One metadata read, not plumbing: the per-layer facts live in the component, and this config block is in a + // different writer from the one that emits the tensors. The table itself is NOT read here (only names, dims + // and the quantised projection header), so this costs a file parse and no memory. + std::string pl_why; + const EmbedPerLayer pl2 = read_embedding_per_layer(pack_dir, p.dims.hidden, &pl_why); + if (pl2.complete()) { + c["hidden_size_per_layer_input"] = pl2.P; + c["vocab_size_per_layer_input"] = pl2.V; + } + } + // The Gemma-4 layer-shape switches the semantics reader also consults, carried through when the pack states them. + { + std::ifstream mc(pack_dir + "/model_config.json"); + if (mc) { + json m; + try { m = json::parse(mc); } catch (...) { m = json::object(); } + const json* t2 = nullptr; + if (m.is_object() && m.contains("config") && m["config"].is_object() && + m["config"].contains("model_attributes") && m["config"]["model_attributes"].is_object() && + m["config"]["model_attributes"].contains("text_config") && + m["config"]["model_attributes"]["text_config"].is_object()) + t2 = &m["config"]["model_attributes"]["text_config"]; + if (t2 != nullptr) + for (const char* k : {"num_kv_shared_layers", "use_double_wide_mlp", "extra_layer_norms"}) + if (t2->contains(k) && !(*t2)[k].is_null()) c[k] = (*t2)[k]; + } + } // A pack that ships its own precomputed RoPE (cos_cache/sin_cache) is not necessarily built for the theta // its genai_config.json states -- pi-inferzo measured 10000 declared against a model that needs 1e6 on // Qwen2.5 rai_1.7.1_hybrid, and our forward computes RoPE from theta rather than reading a cache. That is a From eb93d5f51ccd621b3c1bc744225062ad14e3a548 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 11:17:47 -0400 Subject: [PATCH 12/12] npu-pack: stream the per-layer table, round it to nearest even, read the shared norm (engine#1725) Answers the open review points on #1825: - the banded BF16 conversion is f32_to_bf16 (round to nearest even); it rounded up whenever the kept lsb was 1 - the table is never held: the reader points into the mapped external data and the container writer streams it 32 rows at a time (Writer::add_stream) - one graph parse and one mapping per component, unmapped with the last holder - the config keys come from the Report, so the second whole read is gone Found while writing the import arm: - the projection norm is P wide and shared by the layers; requiring it to be as wide as the table skipped it on a gemma-4 shaped component - the context projection was read at offset 0 of its external data file - an absent zero-point tensor decoded with zero point 0, not the implicit 8 Co-Authored-By: Claude Fable 5.2 --- docs/npu.md | 19 ++++ npu/pack_import.cpp | 240 +++++++++++++++++++++++++------------------- npu/pack_import.h | 42 +++++--- 3 files changed, 183 insertions(+), 118 deletions(-) diff --git a/docs/npu.md b/docs/npu.md index 6862549d4..55abd24a7 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -909,6 +909,25 @@ What converts, and how faithfully: | `model.norm` (final) | `.fconst`, the tail envelope at `layers x stride` | bf16 | byte-exact | | `sin_cache_cos_cache` | `model.onnx.data` | not imported | the lane computes RoPE from `rope_theta` (`fill_rope`) | +**A multi-component pack's per-layer embedding** (engine#1725). gemma-4's embedding component +carries, beside the token table, `embed_tokens_per_layer.weight` (`vocab x layers*P` f32; 262,144 x +8,960 on e2b, 9.4 GiB), a `P`-wide `per_layer_projection_norm.weight` shared by every layer, and +the context projection `per_layer_model_projection.weight_t_Q4` quantised at block 32. The import +writes them under the names the forward loads: + +| container tensor | from | form | +|---|---|---| +| `model.per_layer_token_embd.weight` `[vocab, layers*P]` | the f32 table | BF16, round to nearest even | +| `model.per_layer_proj_norm.weight` `[P]` | the norm | BF16 | +| `model.per_layer_model_proj.weight_layer{l}` `[P, hidden]` | the projection, dequantised, one per layer | BF16 | + +The table is never held in memory, in either form: the reader points into the component's mapped +external data (its extent is checked once, when it is located), and the container writer converts +it 32 rows at a time while it writes the file. A projection with no zero-point tensor takes the +implicit zero point 8, the same rule as every other MatMulNBits here. `config.json` gets +`hidden_size_per_layer_input` and `vocab_size_per_layer_input` only when all three parts went into +the container, because those two keys are what switch the block on in the forward. + An OGA pack keeps its tensors inside the `If` branches: the top-level graph has **no initializers** and four nodes (`Shape`/`Constant`/`Equal`/`If`). The importer walks the subgraphs. The 4-bit decode is `w = (code - zp) * scale` with two codes per byte (low nibble diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 7e8d5c472..75e4b7b0b 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -23,6 +23,8 @@ #include #include +#include +#include #include #include #include @@ -996,6 +998,7 @@ DeclaredMap read_declared(const std::string& pack_dir) { // dtype rule slice 1 established: f32 is read, anything else unquantised is refused by dtype rather than mis-read. // Kept separate from read_embedding_component() because the shapes differ (V x width and a width-wide norm, not V x H) // and because the consumer differs: the token table is the head's ORACLE, these are a decoder INPUT. +void implicit_zp_zero(Q4& q, int bits); // defined below, beside decode_matmulnbits EmbedPerLayer read_embedding_per_layer(const std::string& pack_dir, int64_t hidden, std::string* why) { EmbedPerLayer out; std::string gpath = pack_dir + "/embedding.onnx"; @@ -1036,29 +1039,66 @@ EmbedPerLayer read_embedding_per_layer(const std::string& pack_dir, int64_t hidd return out; } const int64_t V = table->dims.at(0), width = table->dims.at(1); - const auto read_f32 = [&](const OnnxTensor& t, size_t count, const char* what, std::vector* dst) { + // Every external-data file this component names is mapped once and held by `out.keep` (the table's pointer lives in + // one of them); the norm is small and copied out. + struct Maps { + std::vector> all; + std::vector inline_table; + ~Maps() { + for (const auto& [m, n] : all) unmap_file(m, n); + } + }; + auto maps = std::make_shared(); + std::map> by_path; + const auto mapped = [&](const OnnxTensor& t, const char* what, size_t* sz) -> const uint8_t* { const std::string base = fs::path(gpath).parent_path().string() + "/" + t.location; - size_t sz = 0; + const auto it = by_path.find(base); + if (it != by_path.end()) { *sz = it->second.second; return it->second.first; } std::string w2; - const uint8_t* m = map_file(base, sz, &w2); - if (!m) { if (why) *why = std::string(what) + ": " + w2; return false; } + const uint8_t* m = map_file(base, *sz, &w2); + if (!m) { if (why) *why = std::string(what) + ": " + w2; return nullptr; } + maps->all.emplace_back(m, *sz); + by_path[base] = {m, *sz}; + return m; + }; + const auto read_f32 = [&](const OnnxTensor& t, size_t count, const char* what, std::vector* dst) { + size_t sz = 0; + const uint8_t* m = mapped(t, what, &sz); + if (!m) return false; const uint64_t need = uint64_t(count) * 4ull; if (t.offset + need > sz) { if (why) *why = std::string(what) + " declares more bytes than its data carries"; return false; } dst->resize(count); std::memcpy(dst->data(), m + t.offset, size_t(need)); return true; }; + const uint8_t* table_f32 = nullptr; if (!table->location.empty()) { - if (!read_f32(*table, size_t(V) * size_t(width), "embed_tokens_per_layer", &out.table)) return out; + // The table is POINTED AT, not read: V x width f32 is 9.4 GiB on gemma-4-e2b. Its extent is checked here, once, + // so a band read cannot run off the mapping later. + size_t sz = 0; + const uint8_t* m = mapped(*table, "embed_tokens_per_layer", &sz); + if (!m) return out; + if (table->offset + uint64_t(V) * uint64_t(width) * 4ull > sz) { + if (why) *why = "embed_tokens_per_layer declares more bytes than its data carries"; + return out; + } + table_f32 = m + table->offset; } else if (table->raw != nullptr && table->raw_len >= size_t(V) * size_t(width) * 4) { - out.table.resize(size_t(V) * size_t(width)); - std::memcpy(out.table.data(), table->raw, out.table.size() * sizeof(float)); + // an inline initializer is small by construction (it sits inside the graph file); copy it so nothing depends on + // the parsed graph outliving this call + maps->inline_table.resize(size_t(V) * size_t(width)); + std::memcpy(maps->inline_table.data(), table->raw, maps->inline_table.size() * sizeof(float)); + table_f32 = reinterpret_cast(maps->inline_table.data()); } else { if (why) *why = "embed_tokens_per_layer has no readable data"; return out; } - if (norm != nullptr && norm->type == kF32 && norm->dims.size() == 1 && norm->dims.at(0) == width && - !norm->location.empty()) { - if (!read_f32(*norm, size_t(width), "per_layer_projection_norm", &out.norm)) return out; + // The norm is P wide and SHARED by every layer: gemma-4-e2b states it as (256,) beside a table 8960 = 35 x 256 + // wide. Requiring it to be `width` long (as this did) skipped it on exactly that pack, and the loader then refuses + // the container on `model.per_layer_proj_norm.weight`. Any length that divides the table's width is a per-layer + // width; one that does not is not this norm. + if (norm != nullptr && norm->type == kF32 && norm->dims.size() == 1 && norm->dims.at(0) > 0 && + width % norm->dims.at(0) == 0 && !norm->location.empty()) { + if (!read_f32(*norm, size_t(norm->dims.at(0)), "per_layer_projection_norm", &out.norm)) return out; } // The context projection: `per_layer_model_projection.weight_t_Q4` [width][H] at block 32, with `..._t_scales` and // packed `..._t_zero_point`. Dequantised through the file's own MatMulNBits decoder so the arithmetic is the one the @@ -1078,7 +1118,9 @@ EmbedPerLayer read_embedding_per_layer(const std::string& pack_dir, int64_t hidd const auto blob = [&](const OnnxTensor& t, std::string* w4) -> const uint8_t* { if (t.location.empty()) return t.raw; size_t sz = 0; - return map_file(gdir + "/" + t.location, sz, w4); + const uint8_t* m = mapped(t, "per_layer_model_projection", &sz); + if (!m && w4 && why) *w4 = *why; + return m ? m + t.offset : nullptr; }; std::string wa, wb, wc; const uint8_t* qw = blob(*pw, &wa); @@ -1086,8 +1128,11 @@ EmbedPerLayer read_embedding_per_layer(const std::string& pack_dir, int64_t hidd const uint8_t* zp = pz ? blob(*pz, &wc) : nullptr; if (qw && sc) { const int64_t H = hidden; - const Q4 q = decode_matmulnbits(qw, reinterpret_cast(sc), zp, int(width), int(H), 32, - false, zp == nullptr); + Q4 q = decode_matmulnbits(qw, reinterpret_cast(sc), zp, int(width), int(H), 32, + false, zp == nullptr); + // no zero-point tensor means the implicit 2^(bits-1), not 0 (see implicit_zp_zero: ONNX Runtime's rule, + // measured on this family's own embedding component) + if (zp == nullptr) implicit_zp_zero(q, 4); if (q.N == int(width) && q.K == int(H)) { // NL = width / P; the norm is P wide, so P comes from it when the component states one. const int64_t Pn = !out.norm.empty() ? int64_t(out.norm.size()) : width / (width / 256 ? width / 256 : 1); @@ -1115,65 +1160,34 @@ EmbedPerLayer read_embedding_per_layer(const std::string& pack_dir, int64_t hidd } out.V = V; out.width = width; + out.table_f32 = table_f32; + out.keep = maps; if (out.P <= 0) out.P = int64_t(out.norm.size()); // the norm is P wide: it states the per-layer width return out; } -// engine#1725: stream the per-layer token table out in bands. The component states it as f32 (V x NL*P, 9.4 GiB on -// gemma-4-e2b), so materialising it to convert would cost more RAM than the box has to spare for one import; the bands go -// straight from the mapped external data into the container's BF16 layout. -int64_t read_per_layer_table_band(const std::string& pack_dir, EmbedPerLayer* meta, int64_t first_row, int64_t rows, - uint8_t* out, std::string* why) { - if (meta == nullptr || out == nullptr || meta->V <= 0 || meta->width <= 0) { - if (why) *why = "no per-layer table metadata (call read_embedding_per_layer first)"; +// engine#1725: one band of the per-layer token table, f32 -> BF16. The component states the table as f32 (V x NL*P, +// 9.4 GiB on gemma-4-e2b); `meta` already holds the mapping and has checked the table's extent, so this is a bounds +// check on the band and a conversion loop -- `f32_to_bf16`, round to nearest even, the conversion every other tensor in +// the container gets. (It used to re-parse the graph and re-map the data on every call, 8,192 times on e2b, and round +// "up iff the kept lsb is 1", which moved half of all values by one ulp -- including ones bf16 holds exactly.) +int64_t read_per_layer_table_band(const EmbedPerLayer& meta, int64_t first_row, int64_t rows, uint8_t* out, + std::string* why) { + if (out == nullptr || !meta.ok()) { + if (why) *why = "no per-layer table (call read_embedding_per_layer first)"; return -1; } - if (first_row < 0 || rows <= 0 || first_row + rows > meta->V) { + if (first_row < 0 || rows <= 0 || first_row + rows > meta.V) { if (why) *why = "band out of range"; return -1; } - std::string gpath = pack_dir + "/embedding.onnx"; - { - std::ifstream gf(pack_dir + "/genai_config.json"); - if (gf) { - json g; - try { g = json::parse(gf); } catch (...) { g = json::object(); } - if (g.is_object() && g.contains("model") && g["model"].is_object() && g["model"].contains("embedding") && - g["model"]["embedding"].is_object() && g["model"]["embedding"].contains("filename") && - g["model"]["embedding"]["filename"].is_string()) { - const std::string rel = g["model"]["embedding"]["filename"].get(); - if (!rel.empty() && rel.front() != '/' && rel.find("..") == std::string::npos) { - std::error_code nec; - if (fs::is_regular_file(pack_dir + "/" + rel, nec)) gpath = pack_dir + "/" + rel; - } - } - } - } - OnnxModel em; - try { em = parse_onnx(gpath); } catch (const std::exception& e) { if (why) *why = e.what(); return -1; } - const OnnxTensor* table = nullptr; - for (const auto& t : em.tensors) - if (t.name.find("embed_tokens_per_layer") != std::string::npos) table = &t; - if (table == nullptr) { if (why) *why = "the component carries no embed_tokens_per_layer table"; return -1; } - if (table->location.empty()) { if (why) *why = "the per-layer table has no external data (banded reads need it)"; return -1; } - const std::string base = fs::path(gpath).parent_path().string() + "/" + table->location; - size_t sz = 0; - std::string w2; - const uint8_t* m = map_file(base, sz, &w2); - if (!m) { if (why) *why = w2; return -1; } - const uint64_t row_bytes = uint64_t(meta->width) * 4ull; - const uint64_t need_end = table->offset + uint64_t(first_row + rows) * row_bytes; - if (need_end > sz) { if (why) *why = "the per-layer table declares more bytes than its data carries"; return -1; } - const uint8_t* src = m + table->offset + uint64_t(first_row) * row_bytes; - uint16_t* dst = reinterpret_cast(out); - const size_t n = size_t(rows) * size_t(meta->width); + const uint8_t* src = meta.table_f32 + size_t(first_row) * size_t(meta.width) * 4; + const size_t n = size_t(rows) * size_t(meta.width); for (size_t i = 0; i < n; ++i) { float f; std::memcpy(&f, src + i * 4, 4); - uint32_t b; - std::memcpy(&b, &f, 4); - const uint32_t lsb = (b >> 16) & 1u; - dst[i] = uint16_t((b >> 16) + (lsb && ((b & 0xFFFFu) != 0 || (((b >> 16) & 1u) != 0)) ? 1u : 0u)); + const uint16_t h = f32_to_bf16(f); + std::memcpy(out + i * 2, &h, 2); } return rows; } @@ -3387,6 +3401,16 @@ struct Writer { offset += bytes.size(); blobs.emplace_back(name, std::move(bytes)); } + // A tensor too large to hold: `produce` writes exactly `bytes` bytes when the container is written, in table order. + // (engine#1725: gemma-4-e2b's per-layer table is 4.7 GiB as BF16 and comes from a 9.4 GiB f32 mapping.) + void add_stream(const std::string& name, const std::string& dtype, const std::vector& shape, uint64_t bytes, + std::function produce) { + table[name] = {{"dtype", dtype}, {"shape", shape}, {"data_offsets", {offset, offset + bytes}}}; + offset += bytes; + streams[blobs.size()] = {bytes, std::move(produce)}; + blobs.emplace_back(name, std::vector()); + } + std::map>> streams; void finish(const json& metadata) { json t = json::object(); t["__metadata__"] = metadata; @@ -3398,8 +3422,20 @@ struct Writer { const uint64_t len = header.size(); f.write(reinterpret_cast(&len), 8); f.write(header.data(), std::streamsize(header.size())); - for (const auto& [name, bytes] : blobs) - f.write(reinterpret_cast(bytes.data()), std::streamsize(bytes.size())); + for (size_t i = 0; i < blobs.size(); ++i) { + const auto st = streams.find(i); + if (st == streams.end()) { + f.write(reinterpret_cast(blobs[i].second.data()), std::streamsize(blobs[i].second.size())); + continue; + } + const std::streampos before = f.tellp(); + st->second.second(f); + // the header already states this tensor's extent, so a producer that wrote anything else has corrupted + // every offset after it + if (!f || uint64_t(f.tellp() - before) != st->second.first) + throw std::runtime_error(blobs[i].first + ": the streamed tensor did not write the " + + std::to_string(st->second.first) + " B the container's header states"); + } if (!f) throw std::runtime_error("short write into " + dir + "/model.q4nx"); } }; @@ -3876,42 +3912,39 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons // identity = 16 * table[id], per_layer_input = (context + identity)/sqrt(2). Absent for a single-graph pack. { std::string pl_why; - EmbedPerLayer pl = read_embedding_per_layer(pack_dir, H, &pl_why); + const EmbedPerLayer pl = read_embedding_per_layer(pack_dir, H, &pl_why); if (pl.ok()) { - // The token table is streamed: V x (NL*P) f32 is 9.4 GiB on e2b, and a band is 32 rows. - std::vector tbl_bf16(size_t(pl.V) * size_t(pl.width) * 2); - bool bands_ok = true; - for (int64_t row = 0; row < pl.V; row += 32) { - const int64_t n = (pl.V - row < 32) ? (pl.V - row) : 32; - if (read_per_layer_table_band(pack_dir, &pl, row, n, - tbl_bf16.data() + size_t(row) * size_t(pl.width) * 2, &pl_why) != n) { - bands_ok = false; - break; - } - } - if (bands_ok) { - w.add("model.per_layer_token_embd.weight", "BF16", {pl.V, pl.width}, std::move(tbl_bf16)); - if (!pl.norm.empty()) - w.add("model.per_layer_proj_norm.weight", "BF16", {pl.P}, - as_bf16(reinterpret_cast(pl.norm.data()), kF32, pl.norm.size())); - r.tensors += 1 + (pl.norm.empty() ? 0 : 1); - if (pl.complete()) { - for (int64_t l = 0; l < pl.NL; ++l) { - std::vector one(size_t(pl.P) * size_t(pl.H) * 2); - for (int64_t o = 0; o < pl.P; ++o) - for (int64_t k = 0; k < pl.H; ++k) { - const float f = pl.model_proj[size_t(l * pl.P + o) * size_t(pl.H) + size_t(k)]; - uint32_t b; - std::memcpy(&b, &f, 4); - const uint32_t lsb = (b >> 16) & 1u; - reinterpret_cast(one.data())[size_t(o) * size_t(pl.H) + size_t(k)] = - uint16_t((b >> 16) + (lsb && ((b & 0xFFFFu) != 0 || (((b >> 16) & 1u) != 0)) ? 1u : 0u)); - } - w.add("model.per_layer_model_proj.weight_layer" + std::to_string(l), "BF16", {pl.P, pl.H}, - std::move(one)); - ++r.tensors; - } + // The table is STREAMED into the container when it is written: V x (NL*P) f32 is 9.4 GiB on e2b and + // 4.7 GiB as BF16, so neither form is held -- the producer converts 32 rows at a time (573 KB there) + // out of the component's mapping, which the captured `pl` keeps alive until the write. + const uint64_t tbl_bytes = uint64_t(pl.V) * uint64_t(pl.width) * 2ull; + w.add_stream("model.per_layer_token_embd.weight", "BF16", {pl.V, pl.width}, tbl_bytes, + [pl](std::ostream& f) { + std::vector band(size_t(32) * size_t(pl.width) * 2); + std::string bw; + for (int64_t row = 0; row < pl.V; row += 32) { + const int64_t n = (pl.V - row < 32) ? (pl.V - row) : 32; + if (read_per_layer_table_band(pl, row, n, band.data(), &bw) != n) + throw std::runtime_error("model.per_layer_token_embd.weight: " + bw); + f.write(reinterpret_cast(band.data()), + std::streamsize(size_t(n) * size_t(pl.width) * 2)); + } + }); + if (!pl.norm.empty()) + w.add("model.per_layer_proj_norm.weight", "BF16", {pl.P}, + as_bf16(reinterpret_cast(pl.norm.data()), kF32, pl.norm.size())); + r.tensors += 1 + (pl.norm.empty() ? 0 : 1); + if (pl.complete()) { + for (int64_t l = 0; l < pl.NL; ++l) { + w.add("model.per_layer_model_proj.weight_layer" + std::to_string(l), "BF16", {pl.P, pl.H}, + as_bf16(reinterpret_cast(pl.model_proj.data() + + size_t(l) * size_t(pl.P) * size_t(pl.H)), + kF32, size_t(pl.P) * size_t(pl.H))); + ++r.tensors; } + // what switches the block on in the container's config (written further down, by another writer) + r.per_layer_P = pl.P; + r.per_layer_V = pl.V; } } } @@ -4828,16 +4861,11 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons // (q4nx_semantics.c), so a container that carries the per-layer tensors but not these two keys would import them // and never use them -- the forward skips the block silently. Taken from the component itself, which is where they // are measured (gemma-4-e2b: 256 and 262144), rather than from the pack's config, which states them one level down. - { - // One metadata read, not plumbing: the per-layer facts live in the component, and this config block is in a - // different writer from the one that emits the tensors. The table itself is NOT read here (only names, dims - // and the quantised projection header), so this costs a file parse and no memory. - std::string pl_why; - const EmbedPerLayer pl2 = read_embedding_per_layer(pack_dir, p.dims.hidden, &pl_why); - if (pl2.complete()) { - c["hidden_size_per_layer_input"] = pl2.P; - c["vocab_size_per_layer_input"] = pl2.V; - } + // They come from the component, measured when its tensors were written above (gemma-4-e2b: 256 and 262144), and + // are set only when the whole block -- table, norm and context projection -- went into the container. + if (r.per_layer_P > 0 && r.per_layer_V > 0) { + c["hidden_size_per_layer_input"] = r.per_layer_P; + c["vocab_size_per_layer_input"] = r.per_layer_V; } // The Gemma-4 layer-shape switches the semantics reader also consults, carried through when the pack states them. { diff --git a/npu/pack_import.h b/npu/pack_import.h index 9fa0d8608..91a24a4b7 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -35,6 +35,8 @@ #include +#include +#include #include #include @@ -103,7 +105,12 @@ std::vector read_embedding_component(const std::string& pack_dir, std::st // Read separately from the token table because they have different shapes and different consumers: the token table // becomes the head oracle, this becomes an input. Empty (with `why`) when the component has no per-layer table. struct EmbedPerLayer { - std::vector table; // [V][layers*P], row-major (embed_tokens_per_layer.weight) + // [V][layers*P] f32, row-major (embed_tokens_per_layer.weight) -- NOT copied. It is 9.4 GiB on gemma-4-e2b, so the + // reader hands out a pointer into the component's mapped external data and the importer converts it a band at a + // time. `keep` owns whatever that pointer lives in (the mapping, or a copy of an inline initializer) and unmaps it + // when the last copy of this struct goes. + const uint8_t* table_f32 = nullptr; + std::shared_ptr keep; std::vector norm; // [P] (per_layer_projection_norm.weight, shared by every layer) // The CONTEXT projection: the component carries it quantised as // `per_layer_model_projection.weight_t_Q4` [layers*P][H] plus scales and packed zero points. The loader wants it @@ -111,25 +118,34 @@ struct EmbedPerLayer { // -- without it the per-layer input is `16*table/sqrt(2)`, plausible and wrong at every layer (engine#1725). std::vector model_proj; // [layers][P][H] int64_t V = 0, width = 0, P = 0, H = 0, NL = 0; - // `ok()` is what the reader guarantees: the table and the projection norm. `complete()` additionally requires the - // context projection, which the PLE contract needs -- a component with `ok()` but not `complete()` imports the table - // and will be refused by the loader on `per_layer_model_proj.weight_layer0`, which is the honest place to refuse - // (the importer does not know whether the decoder declares per_layer_inputs). + // `ok()` is what the reader guarantees: the table (bounds-checked against its data) and the projection norm. + // `complete()` additionally requires the context projection, which the PLE contract needs -- a component with + // `ok()` but not `complete()` imports the table and will be refused by the loader on + // `per_layer_model_proj.weight_layer0`, which is the honest place to refuse (the importer does not know whether the + // decoder declares per_layer_inputs). bool ok() const { - return V > 0 && width > 0 && table.size() == size_t(V) * size_t(width) && - (!norm.empty() ? norm.size() == size_t(P) : true); + return V > 0 && width > 0 && table_f32 != nullptr && (!norm.empty() ? norm.size() == size_t(P) : true); } bool complete() const { return ok() && P > 0 && H > 0 && NL > 0 && model_proj.size() == size_t(NL) * size_t(P) * size_t(H); } + // one value of the table, for tests and spot checks + float at(int64_t row, int64_t col) const { + float f; + std::memcpy(&f, table_f32 + (size_t(row) * size_t(width) + size_t(col)) * 4, 4); + return f; + } }; // `hidden` is the decoder's hidden size (the context projection's K); it comes from the pack's geometry, not from -// the component, which is why the caller passes it. +// the component, which is why the caller passes it. Parses the component graph and maps its external data ONCE; the +// table's bytes are not read. EmbedPerLayer read_embedding_per_layer(const std::string& pack_dir, int64_t hidden, std::string* why); -// The per-layer token table is `V x (NL*P)` f32 in the component -- 9.4 GiB for gemma-4-e2b -- so it is never -// materialised: the importer streams it into the container in 32-row bands, converting to BF16 as it goes (engine#1725). +// One band of the per-layer token table as BF16 (round to nearest even, the same `f32_to_bf16` every other tensor +// gets), straight from the mapping `meta` holds: no graph parse, no file open, nothing allocated. The importer +// streams the whole table through this in 32-row bands (573 KB at a time on gemma-4-e2b) while the container is +// written, so the table is never held -- not as f32, not as BF16 (engine#1725). // Returns the rows written, or -1 with `why` set. `out` must hold `rows * width * 2` bytes. -int64_t read_per_layer_table_band(const std::string& pack_dir, EmbedPerLayer* meta, int64_t first_row, int64_t rows, - uint8_t* out, std::string* why); +int64_t read_per_layer_table_band(const EmbedPerLayer& meta, int64_t first_row, int64_t rows, uint8_t* out, + std::string* why); // One tensor the pack describes, and where its values would come from. struct Planned { @@ -353,6 +369,8 @@ struct Report { // walk and the pack is tied). Empty means it was read, as usual. Carried on the Report because the config JSON is // built after the .fconst block that discovers this. std::string head_synthesized_note; + // engine#1725: the per-layer embedding block the container carries (0 = none): row width per layer, and rows + int64_t per_layer_P = 0, per_layer_V = 0; bool prefill_tail_seen = false; // a second copy was readable in the prefill .fconst bool final_norm_crosscheck = false; // ... and it is byte-identical to the Token_Token copy