Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 37 additions & 0 deletions npu/pack_import.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3329,6 +3329,9 @@ std::string Plan::manifest() const {
c["role_layer_types"] = {{"keys", container_role_layer_rows},
{"spans", container_role_layer_spans},
{"out_of_range", container_role_layer_out_of_range}};
if (!container_payload_tensors.empty()) c["payload_tensors"] = container_payload_tensors;
if (!container_payload_rows.empty()) c["payload_rows"] = container_payload_rows;
if (!container_role_shapes.empty()) c["role_shapes"] = container_role_shapes;
if (!container_role_layer_span_names.empty())
c["role_layer_types"]["span_names"] = container_role_layer_span_names;
c["note"] = container_note;
Expand Down Expand Up @@ -3647,6 +3650,37 @@ ImportPlan container_import_plan(const Q4nxContainer& container, const std::vect
return out;
}

std::map<int64_t, int64_t> container_payload_tensors_of(const std::vector<Q4nxTensor>& tensors) {
std::map<int64_t, int64_t> out;
for (const auto& t : tensors)
if (t.dtype == "I8" && t.shape.size() == 3) out[t.shape[2]]++;
return out;
}

// the ROWS at each width: what the old name claimed to count
std::map<int64_t, int64_t> container_payload_rows_of(const std::vector<Q4nxTensor>& tensors) {
std::map<int64_t, int64_t> out;
for (const auto& t : tensors)
if (t.dtype == "I8" && t.shape.size() == 3) out[t.shape[2]] += t.shape[0];
return out;
}

std::map<std::string, std::map<std::string, int64_t>> container_role_shapes_of(const std::vector<Q4nxTensor>& tensors) {
std::map<std::string, std::map<std::string, int64_t>> out, by_name;
const ContainerRoles roles = container_roles(tensors);
for (const auto& t : tensors) {
std::string sh = "[";
for (size_t i = 0; i < t.shape.size(); ++i) sh += (i ? "," : "") + std::to_string(t.shape[i]);
by_name[t.name][sh + "] " + t.dtype]++;
}
for (const auto& kv : roles.role_of) {
const auto it = by_name.find(kv.first);
if (it == by_name.end()) continue;
for (const auto& shape : it->second) out[kv.second][shape.first] += shape.second;
}
return out;
}

PlacementReport container_placement(const std::vector<Q4nxTensor>& tensors,
const std::vector<std::string>& layer_types) {
PlacementReport out;
Expand Down Expand Up @@ -4081,6 +4115,9 @@ Plan plan(const std::string& pack_dir) {
if (x.is_string()) lt.push_back(x.get<std::string>());
if (!lt.empty()) {
const RoleLayerReport rep = container_role_layer_types(cc.tensors, lt);
p.container_payload_tensors = container_payload_tensors_of(cc.tensors);
p.container_payload_rows = container_payload_rows_of(cc.tensors);
p.container_role_shapes = container_role_shapes_of(cc.tensors);
p.container_role_layer_rows = int64_t(rep.rows.size());
p.container_role_layer_spans = int64_t(rep.spans.size());
p.container_role_layer_out_of_range = rep.out_of_range;
Expand Down
14 changes: 14 additions & 0 deletions npu/pack_import.h
Original file line number Diff line number Diff line change
Expand Up @@ -267,6 +267,12 @@ struct Plan {
std::map<std::string, int64_t> container_dtypes;
std::string container_note; // what the container genre still needs, empty for a non-container pack
// engine#1788, slice 3c: the (role, layer type) key, when the pack declares layer_types. -1 when it does not.
// engine#1788, 3c residue: the writer's two measurements, kept on the plan because `manifest()` renders from
// members and has no container in scope. `payload_rows` counts the 3-D I8 tensors by payload width (the K the
// fconst tile is built for); `role_shapes` says which shapes each role appears in.
std::map<int64_t, int64_t> container_payload_tensors; // K -> how many 3-D I8 tensors carry that width
std::map<int64_t, int64_t> container_payload_rows; // K -> the ROWS those tensors have, summed over shape[0]
std::map<std::string, std::map<std::string, int64_t>> container_role_shapes;
int64_t container_role_layer_rows = -1;
int64_t container_role_layer_spans = -1;
int64_t container_role_layer_out_of_range = 0;
Expand Down Expand Up @@ -405,6 +411,14 @@ struct RoleSlotReport {
};
RoleSlotReport container_role_slots(const ContainerRoles& roles);

// engine#1788, 3c residue: the two measurements a writer needs, as plain functions over the tensors.
// engine#1788 review: called `payload_rows` while counting TENSORS -- `out[K]++` is one per tensor, and two gate_exps
// tensors are 4096 rows each, so a reader taking the key at its word was off by shape[0] (2713x on the 35B). Both are
// emitted now, each under what it counts.
std::map<int64_t, int64_t> container_payload_tensors_of(const std::vector<Q4nxTensor>& tensors);
std::map<int64_t, int64_t> container_payload_rows_of(const std::vector<Q4nxTensor>& tensors);
std::map<std::string, std::map<std::string, int64_t>> container_role_shapes_of(const std::vector<Q4nxTensor>& tensors);

// engine#1788, slice 3c: the one role whose destination the LAYER TYPE decides, and the reason it decides it. The engine
// carries `linear_attn.qkv_proj.weight` and has NO `self_attn.qkv_proj` slot at all (grep the literals in npu/forward/src),
// so the fused qkv has a destination on a linear layer and nowhere else. Empty string = no destination; the caller must
Expand Down
29 changes: 29 additions & 0 deletions tests/npu_pack_import_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -5550,6 +5550,34 @@ void test_container_write_pack() {
fs::remove_all(dir);
}

// engine#1788, 3c residue: the two measurements a writer needs, armed against the tensors that produce them.
void test_payload_rows_and_role_shapes() {
using pi::Q4nxTensor;
const auto t = [](const std::string& n, std::vector<int64_t> sh, const std::string& dt) {
Q4nxTensor x; x.name = n; x.shape = sh; x.dtype = dt; return x;
};
const std::vector<Q4nxTensor> v = {
t("model.layer.0.mlp.gate_exps_proj.weight", {4096, 8, 4736}, "I8"),
t("model.layer.1.mlp.gate_exps_proj.weight", {4096, 8, 4736}, "I8"),
t("lm_head.weight", {8704, 8, 8704}, "I8"),
t("model.embed_tokens.weight", {248144, 2048}, "BF16"), // 2-D, so neither a tensor row nor a payload row
};
// REVIEW of #2304: the old `payload_rows` counted TENSORS. Both are emitted, and the two numbers must DIFFER.
const auto tb = pi::container_payload_tensors_of(v);
check(tb.size() == 2, "payload_tensors: two distinct widths");
check(tb.at(4736) == 2, "payload_tensors: the two gate_exps tensors share K=4736");
check(tb.count(2048) == 0, "payload_tensors: the 2-D BF16 embedding is not counted");
const auto rb = pi::container_payload_rows_of(v);
check(rb.at(4736) == 2 * 4096, "payload_rows: the rows at that width are 2 x 4096, not 2");
check(rb.at(8704) == 8704, "payload_rows: the lm_head is 8704 rows");
check(rb.at(4736) != tb.at(4736), "payload_rows: the two numbers differ, which is the point");

// role_shapes groups by ROLE and by exact shape+dtype text
const auto rs = pi::container_role_shapes_of(v);
check(rs.count("gate_exps") && rs.at("gate_exps").size() == 1, "role_shapes: the two gate_exps tensors are ONE shape");
check(rs.at("gate_exps").begin()->second == 2, "role_shapes: and it counts both");
}

void test_container_placement() {
using pi::Q4nxTensor;
const auto t = [](const std::string& n) { Q4nxTensor x; x.name = n; x.dtype = "I8"; return x; };
Expand Down Expand Up @@ -6833,6 +6861,7 @@ int main(int argc, char** argv) {
test_container_roles_gemma_local();
test_container_roles_moe_and_singular_layer();
test_container_placement();
test_payload_rows_and_role_shapes();
test_container_write_pack();
test_container_write_layout();
test_container_import_plan();
Expand Down
Loading