Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 78 additions & 0 deletions npu/pack_import.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3325,6 +3325,12 @@ std::string Plan::manifest() const {
c["tensors"] = container_tensors;
c["payload_bytes"] = container_payload_bytes;
c["dtypes"] = container_dtypes;
if (container_role_layer_rows >= 0)
c["role_layer_types"] = {{"keys", container_role_layer_rows},
{"spans", container_role_layer_spans},
{"out_of_range", container_role_layer_out_of_range}};
if (!container_role_layer_span_names.empty())
c["role_layer_types"]["span_names"] = container_role_layer_span_names;
c["note"] = container_note;
j["container"] = c;
}
Expand Down Expand Up @@ -3428,6 +3434,59 @@ std::string header_witness_refusal(int vote_hdrs, bool declared_qk) {
// engine#1786 (item F1), slice 3a: roles from names. The rule is ordered -- the first match wins -- so `q_proj.bias`
// classifies as a bias of q_proj rather than as an unknown tensor, and a name that matches nothing lands in `unassigned`
// with its own name rather than being guessed from its shape.
// engine#1788, slice 3c step 1: (role, layer type), the key the writer needs. See the header for the measurement.
RoleLayerReport container_role_layer_types(const std::vector<Q4nxTensor>& tensors,
const std::vector<std::string>& layer_types) {
RoleLayerReport out;
const ContainerRoles roles = container_roles(tensors);
std::map<std::pair<std::string, std::string>, std::map<std::string, int64_t>> by_key;
std::map<std::string, std::string> role_of_stem;
std::set<std::string> seen_in_layer;
for (const auto& t : tensors) {
const auto it = roles.role_of.find(t.name);
if (it == roles.role_of.end()) continue;
std::string::size_type pos = t.name.find(".layers.");
std::string::size_type off = 8;
if (pos == std::string::npos) { pos = t.name.find(".layer."); off = 7; }
if (pos == std::string::npos) { out.no_layer.push_back(it->second + " (" + t.name + ")"); continue; }
const std::string::size_type start = pos + off;
std::string::size_type end = start;
while (end < t.name.size() && t.name[end] >= '0' && t.name[end] <= '9') ++end;
if (end == start) continue;
const int64_t layer = std::atoll(t.name.substr(start, end - start).c_str());
if (layer < 0 || layer >= int64_t(layer_types.size())) { ++out.out_of_range; continue; }
std::string shape = "[";
for (size_t i = 0; i < t.shape.size(); ++i) shape += (i ? "," : "") + std::to_string(t.shape[i]);
shape += "] " + t.dtype;
// The key is the TENSOR STEM, not the role: `ssm` covers dt.bias, conv1d, norm and a, which are four different
// tensors with four shapes -- a span there is my role vocabulary being coarse, not an ambiguity. Two shapes for
// the SAME stem in ONE layer type is the case a third key would be needed for (gemma4: gate/up/down_proj are
// 2304 wide on 20 layers and 1152 on 15, and both kinds sit in both layer types).
std::string stem = t.name.substr(0, start) + "N" + t.name.substr(end); // `model.layers.7.x` -> `model.layers.N.x`
by_key[{stem, layer_types[size_t(layer)]}][shape]++;
role_of_stem.emplace(stem, it->second);
seen_in_layer.insert(it->second);
}
for (const auto& kv : by_key) {
if (kv.second.size() > 1) {
out.spans.push_back(kv.first.first + " in " + kv.first.second + " appears in " +
std::to_string(kv.second.size()) + " shapes");
continue;
}
int64_t n = 0;
for (const auto& sh : kv.second) n += sh.second;
out.rows.push_back({kv.first.first, role_of_stem[kv.first.first], kv.first.second, kv.second.begin()->first, n});
}
for (const auto& kv : roles.counts) {
if (seen_in_layer.count(kv.first)) continue;
bool already = false;
for (const auto& x : out.no_layer)
if (x.rfind(kv.first + " ", 0) == 0) { already = true; break; }
if (!already) out.no_layer.push_back(kv.first + " (no tensor inside a layer)");
}
return out;
}

ContainerRoles container_roles(const std::vector<Q4nxTensor>& tensors) {
struct Rule { const char* role; const char* needle; };
// Longest/most specific spellings first: `qkv_proj` before `q_proj`, `gate_up_proj` before `gate_proj`.
Expand Down Expand Up @@ -3590,6 +3649,25 @@ Plan plan(const std::string& pack_dir) {
// wins) -- the container's own `dims` still takes precedence in container_geometry().
std::string cfg_source;
const json cfg = load_geometry(pack_dir, &cfg_source);
// engine#1788, slice 3c: the (role, layer type) key, when the pack says what its layers are.
if (const Q4nxContainer& cc = c; true) {
std::vector<std::string> lt;
const json* src = cfg.contains("layer_types") ? &cfg : nullptr;
if (!src && cfg.contains("text_config") && cfg["text_config"].is_object() &&
cfg["text_config"].contains("layer_types"))
src = &cfg["text_config"];
if (src && (*src)["layer_types"].is_array())
for (const auto& x : (*src)["layer_types"])
if (x.is_string()) lt.push_back(x.get<std::string>());
if (!lt.empty()) {
const RoleLayerReport rep = container_role_layer_types(cc.tensors, lt);
p.container_role_layer_rows = int64_t(rep.rows.size());
p.container_role_layer_spans = int64_t(rep.spans.size());
p.container_role_layer_out_of_range = rep.out_of_range;
// the pairs a third key would be needed for, named: this is the writer's remaining question, not a number
for (size_t i = 0; i < rep.spans.size() && i < 8; ++i) p.container_role_layer_span_names.push_back(rep.spans[i]);
}
}
const ContainerGeometry g = container_geometry(c.meta, cfg);
p.dims = g.dims;
p.container_geometry_source = g.source;
Expand Down
26 changes: 26 additions & 0 deletions npu/pack_import.h
Original file line number Diff line number Diff line change
Expand Up @@ -266,6 +266,11 @@ struct Plan {
int64_t container_payload_bytes = 0;
std::map<std::string, int64_t> container_dtypes;
std::string container_note; // what the container genre still needs, empty for a non-container pack
// engine#1788, slice 3c: the (role, layer type) key, when the pack declares layer_types. -1 when it does not.
int64_t container_role_layer_rows = -1;
int64_t container_role_layer_spans = -1;
int64_t container_role_layer_out_of_range = 0;
std::vector<std::string> container_role_layer_span_names; // the pairs a third key would need, named
// engine#1786 (item F1), slice 2: WHERE the container's geometry came from, and what it could not reach.
// A bare number in `dims` cannot say whether the pack declared it or a file beside it did, and those two are free to
// disagree (13 of the 29 packs measured carry a `dims` block; all 29 carry config.json). Empty for a non-container pack.
Expand Down Expand Up @@ -381,6 +386,27 @@ struct ContainerRoles {
};
ContainerRoles container_roles(const std::vector<Q4nxTensor>& tensors);

// engine#1788, slice 3c: a container's roles need a SECOND key, and the second key is the layer type.
// Measured: gemma4-e2b's `q_proj` is N=384 on all 28 `sliding_attention` layers and N=768 on all 7 `full_attention` ones;
// bolt-e4b's is (640,5120) on all 35 sliding and (1280,5120) on all 7 full. A perfect partition, structural in origin
// (5 heads against 10 at head_dim 128). A writer keyed on role alone cannot express these packs; (role, layer type) can.
// A pair that spans TWO shapes is reported in `spans` rather than resolved -- that would need a third key, not a guess.
struct RoleLayerRow {
std::string stem; // the tensor's name with the layer index elided: `model.layers.N.mlp.gate_proj.weight`
std::string role; // that tensor's role, for a reader
std::string layer_type;
std::string shape; // "[a,b,c] I8"
int64_t count = 0;
};
struct RoleLayerReport {
std::vector<RoleLayerRow> rows; // the writer's key, one row per (role, layer type)
std::vector<std::string> spans; // "<role> in <type> appears in N shapes" -- needs a third key
std::vector<std::string> no_layer; // roles with no tensor inside a layer (an embedding, a head, a rope table)
int64_t out_of_range = 0; // tensors whose layer index is outside layer_types
};
RoleLayerReport container_role_layer_types(const std::vector<Q4nxTensor>& tensors,
const std::vector<std::string>& layer_types);

// Reads the pack (model.onnx, its external data, both .fconst headers, config.json).
// Throws only when the pack is structurally unreadable.
// engine#1889 (review of #2063): plan()'s wide-q branch (q_out != hidden), as one function so its effect on the plan is
Expand Down
38 changes: 38 additions & 0 deletions tests/npu_pack_import_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -5140,6 +5140,43 @@ void test_container_plan(const std::string& root) {
// pack that classifies completely, the alternate spellings that must classify the same way, and the two residues (SSM
// and MTP) which must be RECOGNISED rather than reported as unknown, since `unassigned` is only useful if it means
// "nothing knows this".
// engine#1788, slice 3c step 1: (role, layer type), the key the writer needs. Measured on gemma4-e2b (q_proj N=384 on all
// 28 sliding layers, N=768 on all 7 full) and bolt-e4b ((640,5120) on 35 sliding, (1280,5120) on 7 full).
void test_container_role_layer_types() {
const auto t = [](const std::string& n, std::vector<int64_t> shape, const char* dt = "I8") {
pi::Q4nxTensor x; x.name = n; x.dtype = dt; x.shape = std::move(shape); return x;
};
// 2 sliding layers + 1 full layer, q_proj with the sliding/full widths that the real packs show
const std::vector<std::string> lt = {"sliding_attention", "sliding_attention", "full_attention"};
const std::vector<pi::Q4nxTensor> v = {
t("model.layers.0.self_attn.q_proj.weight", {384, 8, 4736}),
t("model.layers.1.self_attn.q_proj.weight", {384, 8, 4736}),
t("model.layers.2.self_attn.q_proj.weight", {768, 8, 4736}),
t("model.embed_tokens.weight", {248144, 2048}, "BF16")};
const pi::RoleLayerReport r = pi::container_role_layer_types(v, lt);
check(r.spans.empty(), "role+layer: the two widths split by layer type, they do not span");
bool sliding = false, full = false;
for (const auto& row : r.rows)
if (row.stem == "model.layers.N.self_attn.q_proj.weight" && row.layer_type == "sliding_attention") {
sliding = row.count == 2 && row.shape == "[384,8,4736] I8";
} else if (row.stem == "model.layers.N.self_attn.q_proj.weight" && row.layer_type == "full_attention") {
full = row.count == 1 && row.shape == "[768,8,4736] I8";
}
check(sliding && full, "role+layer: one row per (role, layer type) with its shape and count");
check(!r.no_layer.empty(), "role+layer: a role outside any layer is reported, not dropped");

// a role carried in two shapes under ONE layer type is the case a third key would be needed for: reported, not resolved
const std::vector<pi::Q4nxTensor> two = {t("model.layers.0.self_attn.q_proj.weight", {384, 8, 4736}),
t("model.layers.1.self_attn.q_proj.weight", {512, 8, 4736})};
const pi::RoleLayerReport r2 = pi::container_role_layer_types(two, lt);
check(r2.spans.size() == 1 && r2.rows.empty(), "role+layer: two shapes in one layer type is a span, not a row");

// a pack whose layer count disagrees with its declared layer_types is counted, not silently clipped
const std::vector<pi::Q4nxTensor> oor = {t("model.layers.9.self_attn.q_proj.weight", {384, 8, 4736})};
check(pi::container_role_layer_types(oor, lt).out_of_range == 1,
"role+layer: a layer index past layer_types is counted as out of range");
}

void test_container_roles() {
const auto t = [](const std::string& n) { pi::Q4nxTensor x; x.name = n; x.dtype = "I8"; return x; };
const auto roles = [&](std::initializer_list<const char*> names) {
Expand Down Expand Up @@ -6181,6 +6218,7 @@ int main(int argc, char** argv) {
test_q4nx_container(golden_dir.empty() ? "." : golden_dir);
test_container_geometry();
test_container_roles();
test_container_role_layer_types();
test_container_plan(golden_dir.empty() ? "." : golden_dir);
test_hybrid_pack_import(golden_dir.empty() ? "." : golden_dir);
test_rope_parameters_theta(golden_dir.empty() ? "." : golden_dir);
Expand Down
Loading