From e183194691cd89fa8e1384501223cf863974aaaa Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 12 Nov 2026 22:53:06 -0400 Subject: [PATCH 1/3] npu-pack: a container's payload widths and the shapes each role appears in, in info (engine#1788, 3c residue) The residue of the closed #2187, rebuilt fresh on main rather than resolved out of ten conflict blocks. Two measurements the writer needs before it can decide whether a tensor is one the engine can consume: payload_rows : counts the 3-D I8 tensors by their payload width (the K the fconst tile is built for) role_shapes : which shapes each role appears in Both are measurements over the container, not mappings, so they cannot be wrong about a destination -- which is why they survive while #2187's placement table did not (main has that table from #2285/#2291). Measured on the 35B: payload_rows {4736: 370, 8704: 1} the one 8704 is lm_head, as its shape says role_shapes 22 roles; down_exps -> [16384,2,4736] I8 x 40 Carried on the Plan and emitted from manifest(), because manifest() is where the container report is rendered and it has no container in scope -- that is the whole reason the first attempt at this port produced two functions with no reader and was reverted. The trip through the plan is the fix, not a style choice. # Conflicts: # npu/pack_import.cpp (cherry picked from commit d7a423080781fd78c5c7ab88c4d8ddbef0036d4f) (cherry picked from commit 3b866fef7294b81740850be4a735533900fff7a6) --- npu/pack_import.cpp | 27 +++++++++++++++++++++++++++ npu/pack_import.h | 9 +++++++++ 2 files changed, 36 insertions(+) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index fcff4a8ed..af6e88299 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -3329,6 +3329,8 @@ std::string Plan::manifest() const { c["role_layer_types"] = {{"keys", container_role_layer_rows}, {"spans", container_role_layer_spans}, {"out_of_range", container_role_layer_out_of_range}}; + if (!container_payload_rows.empty()) c["payload_rows"] = container_payload_rows; + if (!container_role_shapes.empty()) c["role_shapes"] = container_role_shapes; if (!container_role_layer_span_names.empty()) c["role_layer_types"]["span_names"] = container_role_layer_span_names; c["note"] = container_note; @@ -3647,6 +3649,29 @@ ImportPlan container_import_plan(const Q4nxContainer& container, const std::vect return out; } +std::map container_payload_rows_of(const std::vector& tensors) { + std::map out; + for (const auto& t : tensors) + if (t.dtype == "I8" && t.shape.size() == 3) out[t.shape[2]]++; + return out; +} + +std::map> container_role_shapes_of(const std::vector& tensors) { + std::map> out, by_name; + const ContainerRoles roles = container_roles(tensors); + for (const auto& t : tensors) { + std::string sh = "["; + for (size_t i = 0; i < t.shape.size(); ++i) sh += (i ? "," : "") + std::to_string(t.shape[i]); + by_name[t.name][sh + "] " + t.dtype]++; + } + for (const auto& kv : roles.role_of) { + const auto it = by_name.find(kv.first); + if (it == by_name.end()) continue; + for (const auto& shape : it->second) out[kv.second][shape.first] += shape.second; + } + return out; +} + PlacementReport container_placement(const std::vector& tensors, const std::vector& layer_types) { PlacementReport out; @@ -4081,6 +4106,8 @@ Plan plan(const std::string& pack_dir) { if (x.is_string()) lt.push_back(x.get()); if (!lt.empty()) { const RoleLayerReport rep = container_role_layer_types(cc.tensors, lt); + p.container_payload_rows = container_payload_rows_of(cc.tensors); + p.container_role_shapes = container_role_shapes_of(cc.tensors); p.container_role_layer_rows = int64_t(rep.rows.size()); p.container_role_layer_spans = int64_t(rep.spans.size()); p.container_role_layer_out_of_range = rep.out_of_range; diff --git a/npu/pack_import.h b/npu/pack_import.h index f577b6f35..59824306e 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -267,6 +267,11 @@ struct Plan { std::map container_dtypes; std::string container_note; // what the container genre still needs, empty for a non-container pack // engine#1788, slice 3c: the (role, layer type) key, when the pack declares layer_types. -1 when it does not. + // engine#1788, 3c residue: the writer's two measurements, kept on the plan because `manifest()` renders from + // members and has no container in scope. `payload_rows` counts the 3-D I8 tensors by payload width (the K the + // fconst tile is built for); `role_shapes` says which shapes each role appears in. + std::map container_payload_rows; + std::map> container_role_shapes; int64_t container_role_layer_rows = -1; int64_t container_role_layer_spans = -1; int64_t container_role_layer_out_of_range = 0; @@ -405,6 +410,10 @@ struct RoleSlotReport { }; RoleSlotReport container_role_slots(const ContainerRoles& roles); +// engine#1788, 3c residue: the two measurements a writer needs, as plain functions over the tensors. +std::map container_payload_rows_of(const std::vector& tensors); +std::map> container_role_shapes_of(const std::vector& tensors); + // engine#1788, slice 3c: the one role whose destination the LAYER TYPE decides, and the reason it decides it. The engine // carries `linear_attn.qkv_proj.weight` and has NO `self_attn.qkv_proj` slot at all (grep the literals in npu/forward/src), // so the fused qkv has a destination on a linear layer and nowhere else. Empty string = no destination; the caller must From adb1bea58e4137e5805cae05fa86fcd67e6fc924 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 12 Nov 2026 23:07:43 -0400 Subject: [PATCH 2/3] npu-pack: payload_rows counted TENSORS; emit both, and let the numbers differ (review of #2304) `out[K]++` counted one per tensor under a field called `payload_rows`, so on the 35B the old field said 370 at K=4736 where the truth is 1003840 rows -- off by 2713x at exactly the width the fconst tile is built for. Both are emitted now, each under what it counts, and the 8704 row count (7755) independently matches the lm_head's own tensor shape [7755,8,8704]. (cherry picked from commit 5d619cdeb9624391ed2c6f2803cd660de765cdcb) --- npu/pack_import.cpp | 12 +++++++++++- npu/pack_import.h | 7 ++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index af6e88299..106fa8d0c 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -3329,6 +3329,7 @@ std::string Plan::manifest() const { c["role_layer_types"] = {{"keys", container_role_layer_rows}, {"spans", container_role_layer_spans}, {"out_of_range", container_role_layer_out_of_range}}; + if (!container_payload_tensors.empty()) c["payload_tensors"] = container_payload_tensors; if (!container_payload_rows.empty()) c["payload_rows"] = container_payload_rows; if (!container_role_shapes.empty()) c["role_shapes"] = container_role_shapes; if (!container_role_layer_span_names.empty()) @@ -3649,13 +3650,21 @@ ImportPlan container_import_plan(const Q4nxContainer& container, const std::vect return out; } -std::map container_payload_rows_of(const std::vector& tensors) { +std::map container_payload_tensors_of(const std::vector& tensors) { std::map out; for (const auto& t : tensors) if (t.dtype == "I8" && t.shape.size() == 3) out[t.shape[2]]++; return out; } +// the ROWS at each width: what the old name claimed to count +std::map container_payload_rows_of(const std::vector& tensors) { + std::map out; + for (const auto& t : tensors) + if (t.dtype == "I8" && t.shape.size() == 3) out[t.shape[2]] += t.shape[0]; + return out; +} + std::map> container_role_shapes_of(const std::vector& tensors) { std::map> out, by_name; const ContainerRoles roles = container_roles(tensors); @@ -4106,6 +4115,7 @@ Plan plan(const std::string& pack_dir) { if (x.is_string()) lt.push_back(x.get()); if (!lt.empty()) { const RoleLayerReport rep = container_role_layer_types(cc.tensors, lt); + p.container_payload_tensors = container_payload_tensors_of(cc.tensors); p.container_payload_rows = container_payload_rows_of(cc.tensors); p.container_role_shapes = container_role_shapes_of(cc.tensors); p.container_role_layer_rows = int64_t(rep.rows.size()); diff --git a/npu/pack_import.h b/npu/pack_import.h index 59824306e..e3f13acab 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -270,7 +270,8 @@ struct Plan { // engine#1788, 3c residue: the writer's two measurements, kept on the plan because `manifest()` renders from // members and has no container in scope. `payload_rows` counts the 3-D I8 tensors by payload width (the K the // fconst tile is built for); `role_shapes` says which shapes each role appears in. - std::map container_payload_rows; + std::map container_payload_tensors; // K -> how many 3-D I8 tensors carry that width + std::map container_payload_rows; // K -> the ROWS those tensors have, summed over shape[0] std::map> container_role_shapes; int64_t container_role_layer_rows = -1; int64_t container_role_layer_spans = -1; @@ -411,6 +412,10 @@ struct RoleSlotReport { RoleSlotReport container_role_slots(const ContainerRoles& roles); // engine#1788, 3c residue: the two measurements a writer needs, as plain functions over the tensors. +// engine#1788 review: called `payload_rows` while counting TENSORS -- `out[K]++` is one per tensor, and two gate_exps +// tensors are 4096 rows each, so a reader taking the key at its word was off by shape[0] (2713x on the 35B). Both are +// emitted now, each under what it counts. +std::map container_payload_tensors_of(const std::vector& tensors); std::map container_payload_rows_of(const std::vector& tensors); std::map> container_role_shapes_of(const std::vector& tensors); From 8af7e295583eca75b9f203bfed9ac0936870776d Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 12 Nov 2026 23:31:13 -0400 Subject: [PATCH 3/3] tests: arm the two measurements, and that their numbers differ (review of #2304) The arm the review asked for, re-applied by hand after the rebase: at K=4736 the fixture has TWO tensors and 2x4096 = 8192 rows, and the assertion that fails if anyone re-merges the two fields is payload_rows != payload_tensors at that width. --- tests/npu_pack_import_test.cpp | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 7e393254c..33f664795 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -5550,6 +5550,34 @@ void test_container_write_pack() { fs::remove_all(dir); } +// engine#1788, 3c residue: the two measurements a writer needs, armed against the tensors that produce them. +void test_payload_rows_and_role_shapes() { + using pi::Q4nxTensor; + const auto t = [](const std::string& n, std::vector sh, const std::string& dt) { + Q4nxTensor x; x.name = n; x.shape = sh; x.dtype = dt; return x; + }; + const std::vector v = { + t("model.layer.0.mlp.gate_exps_proj.weight", {4096, 8, 4736}, "I8"), + t("model.layer.1.mlp.gate_exps_proj.weight", {4096, 8, 4736}, "I8"), + t("lm_head.weight", {8704, 8, 8704}, "I8"), + t("model.embed_tokens.weight", {248144, 2048}, "BF16"), // 2-D, so neither a tensor row nor a payload row + }; + // REVIEW of #2304: the old `payload_rows` counted TENSORS. Both are emitted, and the two numbers must DIFFER. + const auto tb = pi::container_payload_tensors_of(v); + check(tb.size() == 2, "payload_tensors: two distinct widths"); + check(tb.at(4736) == 2, "payload_tensors: the two gate_exps tensors share K=4736"); + check(tb.count(2048) == 0, "payload_tensors: the 2-D BF16 embedding is not counted"); + const auto rb = pi::container_payload_rows_of(v); + check(rb.at(4736) == 2 * 4096, "payload_rows: the rows at that width are 2 x 4096, not 2"); + check(rb.at(8704) == 8704, "payload_rows: the lm_head is 8704 rows"); + check(rb.at(4736) != tb.at(4736), "payload_rows: the two numbers differ, which is the point"); + + // role_shapes groups by ROLE and by exact shape+dtype text + const auto rs = pi::container_role_shapes_of(v); + check(rs.count("gate_exps") && rs.at("gate_exps").size() == 1, "role_shapes: the two gate_exps tensors are ONE shape"); + check(rs.at("gate_exps").begin()->second == 2, "role_shapes: and it counts both"); +} + void test_container_placement() { using pi::Q4nxTensor; const auto t = [](const std::string& n) { Q4nxTensor x; x.name = n; x.dtype = "I8"; return x; }; @@ -6833,6 +6861,7 @@ int main(int argc, char** argv) { test_container_roles_gemma_local(); test_container_roles_moe_and_singular_layer(); test_container_placement(); + test_payload_rows_and_role_shapes(); test_container_write_pack(); test_container_write_layout(); test_container_import_plan();