diff --git a/CMakeLists.txt b/CMakeLists.txt index d77166dd1..8b0fe5cc3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1002,17 +1002,29 @@ add_test(NAME attn_math COMMAND attn_math_test) # workflow is next touched with that scope and add these to the list itself. `host_gemm_pad_test` is here for the # same reason: without it ctest reports the row as "Not Run" and the job fails after every other test passes. add_dependencies(npu_forward_loader_test hadamard_rotate_test quant_i8_row_test attn_math_test npu_kv_int8_test - host_gemm_pad_test) + host_gemm_pad_test npu_derive_designs_dump) # The window11 gate's comparison policy (engine#834): no tolerance without a measurement behind it, the # statistic named, and a control pair setting the tolerance from its own spread. Shell + the fixtures it # writes itself, so there is no target for the build list to miss. add_test(NAME npu_window11_policy COMMAND bash ${CMAKE_SOURCE_DIR}/scripts/npu-window11.sh --self-test) # The GEMM designs are derived on the container's K grid and the kernel's N grid (engine#845); a # model already on both is unchanged. Pure arithmetic + one assemble_B, so it runs without a model. +# The engine's derived design set for a set of dims, as `ROLE M K N elf` lines. The instrument the +# parity test above needs: the plan generator must state the same kernels q4nx_derive_designs() does, and +# comparing the two is the only way that can be checked rather than assumed (engine#803 A2a). +add_executable(npu_derive_designs_dump tests/npu_derive_designs_dump.c npu/forward/src/q4nx_pack.c) +target_include_directories(npu_derive_designs_dump PRIVATE npu/forward/include) +target_link_libraries(npu_derive_designs_dump PRIVATE m) add_executable(npu_design_grid_test tests/npu_design_grid_test.c npu/forward/src/q4nx_pack.c) target_include_directories(npu_design_grid_test PRIVATE npu/forward/include) target_link_libraries(npu_design_grid_test PRIVATE m) add_test(NAME npu_design_grid COMMAND npu_design_grid_test) +# The plan's `designs` section against the engine's own derivation (engine#803 A2a): the generator and +# q4nx_derive_designs() must state the same kernels, because a section that disagreed would be refused at +# load for every model that carried one. Runs the dump binary, so it is a build dependency above. +add_test(NAME npu_plan_designs_parity + COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/npu_plan_designs_parity_test.py + ${CMAKE_SOURCE_DIR}/tools/npu_plan.py $) # The host fallback GEMM reads a padded design's activations at stride a_k, inside the caller's buffer # (engine#808 review: the split moved the engine#845 pad into the target and left the host branch on the # raw A). The test's activation buffer carries a NaN guard tail, so an implementation that reads past the diff --git a/npu/forward/include/q4nx_pack.h b/npu/forward/include/q4nx_pack.h index cb230d0bf..922caa07f 100644 --- a/npu/forward/include/q4nx_pack.h +++ b/npu/forward/include/q4nx_pack.h @@ -239,6 +239,13 @@ int q4nx_derive_designs(const Q4nxModelDims* d, Q4nxDesignGeom* designs); int q4nx_derive_decode_designs(const Q4nxDesignGeom* designs, int count, Q4nxDesignGeom* out); +/* The UNIQUE name of a design role -- not the display tag the forward logs. + * + * They differ where a model carries two roles that read the same in a log: a qwen3_5 pack derives an + * attention QKV and a GDN QKVLIN, which the forward both prints as "QKV". A plan's `designs[].role` + * is a key (engine#803 A2), so it uses these; the log keeps the short tag. */ +const char* q4nx_design_role_name(Q4nxDesign role); + /* Look up the entry for a role in a derived set; NULL if absent. */ const Q4nxDesignGeom* q4nx_find_design(const Q4nxDesignGeom* designs, int count, Q4nxDesign role); diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index ff9ef1428..0d730e9b9 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -1088,6 +1088,34 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, } + // The plan may name this very set (engine#803 A2a). Checked HERE and not at the end of the + // derivation: the pb16i family appends its split designs and the b8 family rewrites every name + // with the _b8 suffix between the derivation and this loop, so this is the first point at + // which designs_[] is what a target would actually open. When it does, it has to be the set the + // container derives -- same roles, same M/K/N, same artifact -- or the run would take an ELF the + // weights were not written for. A plan without the section agrees trivially, which is what keeps + // every plan written before the field meaningful. + if (plan_.loaded()) { + std::vector derived; + derived.reserve((size_t)ndesigns_); + for (int i = 0; i < ndesigns_; i++) { + onebit::npu::plan::Plan::Design d; + // The plan's role is a key, so it takes the unique name; the log keeps role_tag()'s + // short tag (where QKVLIN prints as "QKV"). + d.role = q4nx_design_role_name(designs_[i].role); + d.M = designs_[i].M; + d.K = designs_[i].K; + d.N = designs_[i].N; + d.elf = designs_[i].elf_name; + derived.push_back(std::move(d)); + } + std::string why; + if (!plan_.designs_agree(derived, why)) { + err_ = "plan " + plan_path_ + ": " + why; + return false; + } + } + for (int i = 0; i < ndesigns_; i++) { const Q4nxDesignGeom* g = &designs_[i]; // The two full-attention-QKV-split designs exist only as pb16i kernels (the diff --git a/npu/forward/src/q4nx_pack.c b/npu/forward/src/q4nx_pack.c index 6ea4e2409..2591eaa80 100644 --- a/npu/forward/src/q4nx_pack.c +++ b/npu/forward/src/q4nx_pack.c @@ -158,6 +158,36 @@ void q4nx_host_gemm_f32_padded(const float* A, int a_k, int M, int K, int N, con free(pad); } +const char* q4nx_design_role_name(Q4nxDesign role) { + switch (role) { + case Q4NX_DESIGN_QKV: return "QKV"; + case Q4NX_DESIGN_QKV2: return "QKV2"; + case Q4NX_DESIGN_O: return "O"; + case Q4NX_DESIGN_O2: return "O2"; + case Q4NX_DESIGN_GU: return "GU"; + case Q4NX_DESIGN_G: return "G"; + case Q4NX_DESIGN_U: return "U"; + case Q4NX_DESIGN_D: return "D"; + case Q4NX_DESIGN_PLEGATE: return "PLEGATE"; + case Q4NX_DESIGN_PLEPROJ: return "PLEPROJ"; + case Q4NX_DESIGN_GU2: return "GU2"; + case Q4NX_DESIGN_D2: return "D2"; + case Q4NX_DESIGN_QKVLIN: return "QKVLIN"; + case Q4NX_DESIGN_Z: return "Z"; + case Q4NX_DESIGN_QA: return "QA"; + case Q4NX_DESIGN_QB: return "QB"; + case Q4NX_DESIGN_KVA: return "KVA"; + case Q4NX_DESIGN_DG: return "DG"; + case Q4NX_DESIGN_DU: return "DU"; + case Q4NX_DESIGN_DD: return "DD"; + case Q4NX_DESIGN_GB: return "GB"; + case Q4NX_DESIGN_UB: return "UB"; + case Q4NX_DESIGN_QKVFA_Q: return "QKVFAQ"; + case Q4NX_DESIGN_QKVFA_KV: return "QKVFAKV"; + default: return "?"; + } +} + /* One standard design: the model's real K and N in, the kernel's grids out, the real widths kept * beside them. Role-specific row counts (q_rows/k_rows/v_rows, gate_rows/up_rows) are the * caller's, because only it knows what fills N. MLA keeps its own derivation: its KVA pad is the diff --git a/npu/plan/plan.h b/npu/plan/plan.h index 2bb9df51c..80b6cb7d7 100644 --- a/npu/plan/plan.h +++ b/npu/plan/plan.h @@ -161,6 +161,39 @@ struct Plan { struct { bool enabled = false; int K = 0, N = 0; } moe_batched_gu; std::vector layer_plans; + // One kernel the plan expects: the role it serves, the geometry it launches at and the artifact + // that implements it. `role` is the forward's own role tag ("QKV", "O", "GU", "G", "U", "D", + // "GB", "UB", the Gemma4 second attention type, the MLA and GDN roles), so a reader can match a + // design to the code that launches it without a second name table. + // + // It is the unique name (q4nx_design_role_name(): "QKV", "QKVLIN", "QA", "GB", ...), NOT the + // forward's short log tag -- those differ exactly where a model carries two roles that read alike, + // since a qwen3_5 pack derives an attention QKV and a GDN QKVLIN and both log as "QKV". + // designs_agree() still compares a multiset of (role, M, K, N, elf) rather than a role-keyed + // lookup: that is what makes a duplicated or mispaired set a refusal instead of a match, and it + // keeps the check honest if a future role name is ever reused. + // + // Why the plan carries this at all (engine#803 A2): the design set is what a target opens, and + // while it was derived in the forward, "which ELF does this role run" was answered by + // q4nx_derive_designs() plus filename probes inside one file. A plan that names the set makes + // the kernel identity part of the model's description -- and the engine then refuses a plan whose + // designs are not the ones the container derives, instead of silently running whatever the probes + // find. An absent section is the old behaviour (the probes), so every plan written before this + // field existed keeps loading. + struct Design { + std::string role; + int M = 0, K = 0, N = 0; + std::string elf; + }; + std::vector designs; + + // True when the plan's `designs` and a derived set name the same kernels: same roles, same M/K/N, + // same artifact, both directions (a role the plan omits is a kernel the model needs and the plan + // would not open). An empty section agrees with anything. On disagreement `err` names the role + // and both sides -- never a default, because a plan that names a different ELF than the weights + // need is exactly the silent divergence this layer exists to remove. + bool designs_agree(const std::vector& derived, std::string& err) const; + bool loaded() const { return !format.empty(); } // Load and validate. Throws PlanError naming the field for anything this build cannot run -- @@ -351,6 +384,32 @@ struct Plan { if (ternary_tag != p.pb16i) throw PlanError("plan: " + path + " kernels.tag \"" + p.tag + "\" and container.pb16i " + (p.pb16i ? "true" : "false") + " disagree"); + // The per-role kernel set (engine#803 A2). Optional: absent means the engine keeps deriving it + // from the container's dims, which is what every plan written before this field means. + if (j.contains("designs")) { + const auto& ds = j.at("designs"); + if (!ds.is_array()) throw PlanError("plan: " + path + " designs is not an array"); + for (size_t i = 0; i < ds.size(); i++) { + const auto& d = ds.at(i); + const std::string where = "designs[" + std::to_string(i) + "]"; + if (!d.is_object()) throw PlanError("plan: " + path + " " + where + " is not an object"); + Design des; + des.role = d.value("role", ""); + if (des.role.empty()) throw PlanError("plan: " + path + " " + where + ".role is empty"); + // Named with the index, not just the field: a plan can carry a dozen of these and + // "M must be positive" without a position is a hunt. + des.M = d.value("M", 0); + if (des.M <= 0) throw PlanError("plan: " + path + " " + where + ".M must be positive"); + des.K = d.value("K", 0); + if (des.K <= 0) throw PlanError("plan: " + path + " " + where + ".K must be positive"); + des.N = d.value("N", 0); + if (des.N <= 0) throw PlanError("plan: " + path + " " + where + ".N must be positive"); + des.elf = d.value("elf", ""); + if (des.elf.empty()) + throw PlanError("plan: " + path + " " + where + ".elf is empty (the artifact the role runs)"); + p.designs.push_back(des); + } + } return p; } @@ -397,4 +456,56 @@ struct Plan { } }; +// Out of line for readability: the comparison is the whole point of the section. `derived` is what +// the container's own geometry produces (q4nx_derive_designs() plus the MoE pair), so this is the +// check that keeps a plan and the artifacts its weights need from drifting apart silently. +inline bool Plan::designs_agree(const std::vector& derived, std::string& err) const { + if (designs.empty()) return true; // no section: the engine derives, as it always did + // Compared as a MULTISET of (role, M, K, N, elf), not by role alone: the role tags are not unique -- + // a qwen3_5 pack derives both an attention QKV and a GDN QKVLIN design, and both are logged as + // "QKV" -- so a role-keyed lookup would compare the wrong pair and, worse, report agreement. + auto same = [](const Design& a, const Design& b) { + return a.role == b.role && a.M == b.M && a.K == b.K && a.N == b.N && a.elf == b.elf; + }; + std::vector used(derived.size(), false); + for (const auto& mine : designs) { + bool found = false; + for (size_t i = 0; i < derived.size(); i++) { + if (used[i] || !same(mine, derived[i])) continue; + used[i] = found = true; + break; + } + if (found) continue; + // Not found: is it a role the container does not derive at all, or the same role at a different + // geometry / artifact? The second is the dangerous one, so it says both sides. + const Design* near = nullptr; + for (const auto& d : derived) + if (d.role == mine.role) near = &d; + if (!near) { + err = "plan designs: role \"" + mine.role + "\" is not a design this container derives"; + return false; + } + if (near->M != mine.M || near->K != mine.K || near->N != mine.N) { + err = "plan designs(" + mine.role + "): M=" + std::to_string(mine.M) + " K=" + + std::to_string(mine.K) + " N=" + std::to_string(mine.N) + ", the container derives M=" + + std::to_string(near->M) + " K=" + std::to_string(near->K) + " N=" + + std::to_string(near->N); + return false; + } + err = "plan designs(" + mine.role + "): elf \"" + mine.elf + "\", the container derives \"" + + near->elf + "\""; + return false; + } + // Both directions: a derived design the plan does not name is a kernel the model needs and the plan + // would not open once the target takes its set from here. + for (size_t i = 0; i < derived.size(); i++) { + if (!used[i]) { + err = "plan designs: the container derives " + derived[i].role + " (" + derived[i].elf + + ") but the plan does not name it"; + return false; + } + } + return true; +} + } // namespace onebit::npu::plan diff --git a/tests/npu_derive_designs_dump.c b/tests/npu_derive_designs_dump.c new file mode 100644 index 000000000..f85317c9b --- /dev/null +++ b/tests/npu_derive_designs_dump.c @@ -0,0 +1,78 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// npu_derive_designs_dump -- print the design set the engine derives for a set of model dims. +// +// npu_derive_designs_dump H NH NKV HD IM fused_gu [attn_output_gate] [tag] +// +// One line per design: `ROLE M K N elf`, with the unique role name (q4nx_design_role_name()), the +// geometry and the artifact. It exists so the plan generator can be checked against the derivation +// instead of against a second copy of the rules: `tests/npu_plan_designs_parity_test.py` runs this +// binary and tools/npu_plan.py on the same dims and requires the two sets to be identical, which is +// the only way a plan's `designs` section (engine#803 A2a) can be trusted to name the kernels the +// container's weights were written for. +// +// No model, no device: the derivation is pure arithmetic over the dims. +#include "q4nx_pack.h" + +#include +#include +#include + +int main(int argc, char** argv) { + if (argc != 7 && argc != 8 && argc != 9) { + fprintf(stderr, "usage: %s H NH NKV HD IM fused_gu [attn_output_gate] [tag]\n", argv[0]); + return 2; + } + Q4nxModelDims d; + memset(&d, 0, sizeof(d)); + d.H = atoi(argv[1]); + d.NH = atoi(argv[2]); + d.NKV = atoi(argv[3]); + d.HD = atoi(argv[4]); + d.IM = atoi(argv[5]); + d.fused_gu = atoi(argv[6]); + d.attn_output_gate = argc > 7 ? atoi(argv[7]) : 0; + d.M = Q4NX_DESIGN_M_PREFILL; + /* The forward rewrites every artifact name with "_b8" for a b8 container (the blocked 8x8 tile + * layout the kernels carry); the derivation itself does not know the layout, so the dump applies + * the same rewrite when asked. Absent tag = the rm/pb16i/t2s families, whose prefill names are the + * untagged int8 ones. */ + const char* tag = argc > 8 ? argv[8] : ""; + + Q4nxDesignGeom designs[Q4NX_MAX_DESIGNS]; + memset(designs, 0, sizeof(designs)); + const int n = q4nx_derive_designs(&d, designs); + if (n <= 0) { + fprintf(stderr, "derive failed for H=%d NH=%d NKV=%d HD=%d IM=%d fused_gu=%d\n", d.H, d.NH, + d.NKV, d.HD, d.IM, d.fused_gu); + return 1; + } + for (int i = 0; i < n; i++) { + char name[Q4NX_ELF_NAME_MAX]; + snprintf(name, sizeof(name), "%s", designs[i].elf_name); + if (strcmp(tag, "_b8") == 0 || strcmp(tag, "b8") == 0) { + const size_t len = strlen(name); + if (len > 4 && strcmp(name + len - 4, ".elf") == 0) { + char tagged[Q4NX_ELF_NAME_MAX]; + snprintf(tagged, sizeof(tagged), "%.*s_b8.elf", (int)(len - 4), name); + snprintf(name, sizeof(name), "%s", tagged); + } + } + printf("%s %d %d %d %s\n", q4nx_design_role_name(designs[i].role), designs[i].M, designs[i].K, + designs[i].N, name); + } + return 0; +} diff --git a/tests/npu_plan_designs_parity_test.py b/tests/npu_plan_designs_parity_test.py new file mode 100755 index 000000000..c2a231d31 --- /dev/null +++ b/tests/npu_plan_designs_parity_test.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""tests/npu_plan_designs_parity_test.py -- the plan's `designs` section against the engine's own derivation. + +engine#803 A2a: a plan may name the per-role kernel set, and the engine refuses a plan that disagrees with +what the container derives. That check is only useful if the GENERATOR states the same set -- so this test +runs both sides on the same dims and requires identical sets: + + * tools/npu_plan.py's designs_from_config() (Python, what a plan will carry) + * npu_derive_designs_dump (the engine's q4nx_derive_designs(), the same call the forward makes) + +A disagreement here is worse than a missing section: the section is checked both ways, so a generator that +named a different ELF would refuse every plan-carrying model at load. The last part of the test pins the +other half -- the families this tool does not model yet must emit NO section at all, so they keep loading +with the engine's derivation. + +Usage: npu_plan_designs_parity_test.py +""" +import importlib.util +import sys + +TOOL, DUMP = sys.argv[1], sys.argv[2] +failures = [] + + +def check(ok, what): + if not ok: + print(f"FAIL {what}") + failures.append(what) + + +spec = importlib.util.spec_from_file_location("npu_plan", TOOL) +npu_plan = importlib.util.module_from_spec(spec) +spec.loader.exec_module(npu_plan) + + +def engine_designs(cfg) -> list[str]: + """The engine's set for this config, as `ROLE M K N elf` lines.""" + import subprocess + + argv = [DUMP, str(cfg["hidden_size"]), str(cfg["num_attention_heads"]), + str(cfg.get("num_key_value_heads", cfg["num_attention_heads"])), + str(cfg.get("head_dim") or cfg["hidden_size"] // cfg["num_attention_heads"]), + str(cfg["intermediate_size"]), "1" if cfg.get("_fused_gu", True) else "0", + "1" if cfg.get("attn_output_gate") else "0", cfg.get("_tag", "")] + out = subprocess.run(argv, capture_output=True, text=True) + if out.returncode != 0: + raise SystemExit(f"{DUMP} failed: {out.stderr.strip()}") + return [ln for ln in out.stdout.splitlines() if ln.strip()] + + +def plan_designs(cfg) -> list[str] | None: + got = npu_plan.designs_from_config(cfg, cfg.get("_tag", ""), cfg.get("_fused_gu", True)) + if got is None: + return None + return [f"{d['role']} {d['M']} {d['K']} {d['N']} {d['elf']}" for d in got] + + +# The matrix: dense configs only, in the shapes the derivation's rules branch on -- the 0.6B's own dims, a +# padded hidden (896 -> K 1024, the case engine#845 exists for), q|gate doubling, a split gate/up, odd head +# counts, and a b8 family suffix. +DENSE = [ + {"name": "qwen3-0.6b", "hidden_size": 1024, "intermediate_size": 3072, "num_attention_heads": 16, + "num_key_value_heads": 8, "head_dim": 128}, + {"name": "0.6b split gate/up", "hidden_size": 1024, "intermediate_size": 3072, "num_attention_heads": 16, + "num_key_value_heads": 8, "head_dim": 128, "_fused_gu": False}, + {"name": "padded hidden 896", "hidden_size": 896, "intermediate_size": 4864, "num_attention_heads": 14, + "num_key_value_heads": 2, "head_dim": 64}, + {"name": "attn_output_gate", "hidden_size": 1024, "intermediate_size": 3072, "num_attention_heads": 16, + "num_key_value_heads": 8, "head_dim": 128, "attn_output_gate": True}, + {"name": "no head_dim key (derived)", "hidden_size": 512, "intermediate_size": 2048, + "num_attention_heads": 8, "num_key_value_heads": 8}, + {"name": "b8 family", "hidden_size": 1024, "intermediate_size": 3072, "num_attention_heads": 16, + "num_key_value_heads": 8, "head_dim": 128, "_tag": "_b8"}, +] + +for case in DENSE: + name = case.pop("name") + want = engine_designs(case) + got = plan_designs(case) + check(got is not None, f"{name}: the generator states a set") + if got is None: + continue + check(got == want, f"{name}: generator set == engine set\n engine: {want}\n plan: {got}") + +# Families the tool does not model yet: NO section, so the engine keeps deriving and the model still loads. +OMIT = [ + ("mla", {"hidden_size": 1024, "intermediate_size": 3072, "num_attention_heads": 16, + "num_key_value_heads": 1, "head_dim": 128, "q_lora_rank": 1536, "kv_lora_rank": 512}), + ("gdn layer mix", {"hidden_size": 1024, "intermediate_size": 3072, "num_attention_heads": 16, + "num_key_value_heads": 8, "head_dim": 128, + "linear_num_key_heads": 4, "linear_key_head_dim": 128, + "linear_num_value_heads": 8, "linear_value_head_dim": 128, + "layer_types": ["linear_attention", "full_attention"]}), + ("second attention type", {"hidden_size": 1024, "intermediate_size": 3072, "num_attention_heads": 16, + "num_key_value_heads": 8, "head_dim": 128, "global_head_dim": 256}), + ("per-layer embeddings", {"hidden_size": 1024, "intermediate_size": 3072, "num_attention_heads": 16, + "num_key_value_heads": 8, "head_dim": 128, "ple_dim": 32}), + ("double-wide mlp", {"hidden_size": 1024, "intermediate_size": 3072, "num_attention_heads": 16, + "num_key_value_heads": 8, "head_dim": 128, "use_double_wide_mlp": True}), +] +for name, cfg in OMIT: + check(plan_designs(cfg) is None, f"{name}: no section is emitted (the engine keeps deriving)") + +print("npu_plan_designs_parity: " + ("ok" if not failures else f"{len(failures)} failure(s)")) +sys.exit(1 if failures else 0) diff --git a/tests/npu_plan_test.cpp b/tests/npu_plan_test.cpp index d05b0a150..6d0e91e83 100644 --- a/tests/npu_plan_test.cpp +++ b/tests/npu_plan_test.cpp @@ -360,11 +360,122 @@ void test_kv() { } } +// ---- designs: the per-role kernel set the plan may name (engine#803 A2) ---- +// +// Two halves, both device-free: the loader accepts and rejects the section by field, and +// designs_agree() is the comparison the forward runs against what the container derives. +void test_designs() { + auto with_designs = [](bool second) { + auto p = base_plan(); + p["designs"] = nlohmann::json::array(); + p["designs"].push_back({{"role", "QKV"}, {"M", 128}, {"K", 1024}, {"N", 2048}, + {"elf", "full_i8_QKV_K1024_N2048.elf"}}); + if (second) + p["designs"].push_back({{"role", "O"}, {"M", 128}, {"K", 2048}, {"N", 1024}, + {"elf", "full_i8_O_K2048_N1024.elf"}}); + return p; + }; + const Plan p = Plan::load(write_plan("designs2", with_designs(true))); + check(p.designs.size() == 2, "designs: two entries parsed"); + check(p.designs[0].role == "QKV" && p.designs[0].M == 128 && p.designs[0].K == 1024 && + p.designs[0].N == 2048 && p.designs[0].elf == "full_i8_QKV_K1024_N2048.elf", + "designs: role/M/K/N/elf are what the plan says"); + + // A fresh derived set per case, so a case cannot inherit an earlier mutation. + auto derived = [] { + return std::vector{ + {"QKV", 128, 1024, 2048, "full_i8_QKV_K1024_N2048.elf"}, + {"O", 128, 2048, 1024, "full_i8_O_K2048_N1024.elf"}, + }; + }; + std::string why; + check(p.designs_agree(derived(), why), "designs: the same set agrees"); + { + auto swapped = derived(); + std::swap(swapped[0], swapped[1]); + check(p.designs_agree(swapped, why), "designs: the comparison is by role, not by position"); + } + // The refusal that matters: same shape, wrong artifact -- the ELF the weights were not written for. + { + auto d = derived(); + d[1].elf = "full_i8_O_K2048_N1024_b8.elf"; + why.clear(); + check(!p.designs_agree(d, why), "designs: a different artifact is refused"); + check(has(why, "O") && has(why, "_b8"), + "designs: the artifact refusal names the role and the file: " + why); + } + { + auto d = derived(); + d[1].N = 512; + why.clear(); + check(!p.designs_agree(d, why), "designs: a different geometry is refused"); + check(has(why, "N=512") && has(why, "N=1024"), + "designs: the geometry refusal carries both sides: " + why); + } + // Both directions: a role the container derives and the plan omits is a kernel the plan would not open. + { + auto d = derived(); + d.push_back({"D", 128, 3072, 1024, "full_i8_D_K3072_N1024.elf"}); + why.clear(); + check(!p.designs_agree(d, why), "designs: a derived role the plan omits is refused"); + check(has(why, "D"), "designs: the omission names the role: " + why); + } + + // The role tags are NOT unique: a qwen3_5 pack derives an attention QKV and a GDN QKVLIN design, + // and both are logged as "QKV". A role-keyed comparison would compare the wrong pair, so the + // agreement is a multiset of (role, M, K, N, elf) and this case is the one that proves it. + { + auto two_qkv = with_designs(true); + two_qkv["designs"] = nlohmann::json::array({ + {{"role", "QKV"}, {"M", 128}, {"K", 1024}, {"N", 4096}, {"elf", "full_i8_QKV_K1024_N4096.elf"}}, + {{"role", "QKV"}, {"M", 128}, {"K", 1024}, {"N", 2048}, {"elf", "full_i8_QKVLIN_K1024_N2048.elf"}}, + }); + const Plan q = Plan::load(write_plan("twosame", two_qkv)); + std::string w; + check(q.designs_agree({{"QKV", 128, 1024, 4096, "full_i8_QKV_K1024_N4096.elf"}, + {"QKV", 128, 1024, 2048, "full_i8_QKVLIN_K1024_N2048.elf"}}, + w), + "designs: two roles that share a tag are matched as a multiset"); + // Swapping the geometries must be refused: same role, wrong one-to-one pairing. + w.clear(); + check(!q.designs_agree({{"QKV", 128, 1024, 2048, "full_i8_QKVLIN_K1024_N2048.elf"}, + {"QKV", 128, 1024, 4096, "full_i8_QKV_K1024_N4096.elf"}, + {"QKV", 128, 1024, 4096, "full_i8_QKV_K1024_N4096.elf"}}, + w), + "designs: a duplicated geometry cannot stand in for an omitted one"); + } + // No section: the probes' behaviour, and every plan written before the field keeps loading. + const Plan plain = Plan::load(write_plan("nodesigns", base_plan())); + check(plain.designs.empty(), "designs: an absent section parses to empty"); + why.clear(); + check(plain.designs_agree({{"QKV", 128, 1024, 2048, "x.elf"}}, why), + "designs: an absent section agrees with anything"); + + // Refusals, each naming the field. + { + auto bad = with_designs(false); + bad["designs"][0]["elf"] = ""; + check(has(refusal(write_plan("dnoelf", bad)), ".elf is empty"), "designs: an empty elf is refused"); + } + { + auto bad = with_designs(false); + bad["designs"][0]["M"] = 0; + check(has(refusal(write_plan("dnom", bad)), "designs[0]"), "designs: a non-positive M is refused by index"); + } + { + auto bad = with_designs(false); + bad["designs"] = 7; + check(has(refusal(write_plan("dnotarray", bad)), "designs is not an array"), + "designs: a non-array section is refused"); + } +} + int main() { test_valid(); test_kv(); test_tile_order(); test_refusals(); + test_designs(); std::printf("%s\n", failures ? "FAILED" : "npu_plan: ok"); return failures ? 1 : 0; } diff --git a/tools/npu_plan.py b/tools/npu_plan.py index f5891b41b..a9d0a143b 100644 --- a/tools/npu_plan.py +++ b/tools/npu_plan.py @@ -50,6 +50,11 @@ N_GRID = 512 +def mla_config(cfg: dict) -> bool: + """The MLA family, which the C side flags from the same keys (q4nx_pack.c derive_mla's caller).""" + return bool(cfg.get("q_lora_rank") or cfg.get("kv_lora_rank")) + + def pad_k(k: int) -> int: return (k + K_GRID - 1) // K_GRID * K_GRID @@ -303,9 +308,71 @@ def generate(model_dir: str, kernels: str) -> dict: "moe": {"batched_expert_gu": moe_batched(cfg, hidden, model_dir, kernels)}, "layers": layers, } + # The per-role kernel set (engine#803 A2a). Written only when this tool can state the WHOLE set: + # the plan's section is checked both ways, so a partial one would be refused at load and a missing + # one is exactly today's behaviour (the engine derives it). + designs = designs_from_config(cfg, prefill_tag if layout == "b8" else "", fused_gu) + if designs is not None: + plan["designs"] = designs return plan +def designs_from_config(cfg: dict, prefill_tag: str, fused_gu: bool) -> list[dict] | None: + """The engine's per-role kernel set for this config, or None when this tool cannot state it fully. + + These are the rules q4nx_derive_designs() applies (npu/forward/src/q4nx_pack.c), ported here so a + plan can NAME the set instead of the engine re-deriving it (engine#803 A2a). The port is checked + against the C derivation by tests/npu_plan_designs_parity_test.py, which runs both on the same + dims and requires identical sets -- the reason this function exists as a function. + + WHY IT RETURNS None FOR SOME MODELS: the plan's `designs` section is checked BOTH ways, so a + section that omitted a role the container derives would be refused at load. The families below + derive roles this tool does not model yet (MLA's q_a/q_b/kv_a, Gemma4's second attention type and + per-layer embeddings and double-wide MLP, GDN's fused q|k|v / z / ssm_out), and for those the right + output is NO section -- absent means the engine keeps deriving, which is what every plan written + before this field means. Emitting a partial set would be worse than emitting nothing. + """ + if mla_config(cfg) or cfg.get("layer_types") or cfg.get("global_head_dim") or cfg.get("ple_dim") \ + or cfg.get("use_double_wide_mlp") or cfg.get("linear_num_key_heads"): + return None + hidden = cfg.get("hidden_size") + im = cfg.get("intermediate_size") + nh = cfg.get("num_attention_heads", 0) + nkv = cfg.get("num_key_value_heads", nh) + hd = cfg.get("head_dim") or (hidden // nh if nh else 0) + if not (hidden and im and nh and hd and nkv): + return None + + def elf(tag: str, k: int, n: int) -> str: + return f"full_i8_{tag}_K{pad_k(k)}_N{pad_n(n)}{prefill_tag}.elf" + + # q_rows is the attention width WITHOUT the output gate: attn_output_gate doubles what the q + # projection emits (so it doubles QKV's N), but O still reads one head width per head + # (q4nx_derive_designs: `q_rows = NH*HD` for O's K, `q_rows * (gate?2:1)` for QKV's N). + q_rows = nh * hd + qkv_rows = q_rows * (2 if cfg.get("attn_output_gate") else 1) + kv_rows = nkv * hd + out = [ + {"role": "QKV", "M": 128, "K": pad_k(hidden), "N": pad_n(qkv_rows + 2 * kv_rows), + "elf": elf("QKV", hidden, qkv_rows + 2 * kv_rows)}, + {"role": "O", "M": 128, "K": pad_k(q_rows), "N": pad_n(hidden), + "elf": elf("O", q_rows, hidden)}, + ] + # gate/up is a plan decision the caller already made (the artifact probe above), so it is passed in + # rather than probed a second time here: two copies of that rule is how a plan and its images drift. + if fused_gu: + out.append({"role": "GU", "M": 128, "K": pad_k(hidden), "N": pad_n(2 * im), + "elf": elf("GU", hidden, 2 * im)}) + else: + out.append({"role": "G", "M": 128, "K": pad_k(hidden), "N": pad_n(im), + "elf": elf("G", hidden, im)}) + out.append({"role": "U", "M": 128, "K": pad_k(hidden), "N": pad_n(im), + "elf": elf("U", hidden, im)}) + out.append({"role": "D", "M": 128, "K": pad_k(im), "N": pad_n(hidden), + "elf": elf("D", im, hidden)}) + return out + + def moe_batched(cfg: dict, hidden: int, model_dir: str, kernels: str) -> dict: """The batched-expert gate/up kernel, when the model has experts and the kernel exists.""" experts = cfg.get("num_experts") or cfg.get("n_routed_experts") or 0