Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1002,17 +1002,29 @@ add_test(NAME attn_math COMMAND attn_math_test)
# workflow is next touched with that scope and add these to the list itself. `host_gemm_pad_test` is here for the
# same reason: without it ctest reports the row as "Not Run" and the job fails after every other test passes.
add_dependencies(npu_forward_loader_test hadamard_rotate_test quant_i8_row_test attn_math_test npu_kv_int8_test
host_gemm_pad_test)
host_gemm_pad_test npu_derive_designs_dump)
# The window11 gate's comparison policy (engine#834): no tolerance without a measurement behind it, the
# statistic named, and a control pair setting the tolerance from its own spread. Shell + the fixtures it
# writes itself, so there is no target for the build list to miss.
add_test(NAME npu_window11_policy COMMAND bash ${CMAKE_SOURCE_DIR}/scripts/npu-window11.sh --self-test)
# The GEMM designs are derived on the container's K grid and the kernel's N grid (engine#845); a
# model already on both is unchanged. Pure arithmetic + one assemble_B, so it runs without a model.
# The engine's derived design set for a set of dims, as `ROLE M K N elf` lines. The instrument the
# parity test above needs: the plan generator must state the same kernels q4nx_derive_designs() does, and
# comparing the two is the only way that can be checked rather than assumed (engine#803 A2a).
add_executable(npu_derive_designs_dump tests/npu_derive_designs_dump.c npu/forward/src/q4nx_pack.c)
target_include_directories(npu_derive_designs_dump PRIVATE npu/forward/include)
target_link_libraries(npu_derive_designs_dump PRIVATE m)
add_executable(npu_design_grid_test tests/npu_design_grid_test.c npu/forward/src/q4nx_pack.c)
target_include_directories(npu_design_grid_test PRIVATE npu/forward/include)
target_link_libraries(npu_design_grid_test PRIVATE m)
add_test(NAME npu_design_grid COMMAND npu_design_grid_test)
# The plan's `designs` section against the engine's own derivation (engine#803 A2a): the generator and
# q4nx_derive_designs() must state the same kernels, because a section that disagreed would be refused at
# load for every model that carried one. Runs the dump binary, so it is a build dependency above.
add_test(NAME npu_plan_designs_parity
COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/npu_plan_designs_parity_test.py
${CMAKE_SOURCE_DIR}/tools/npu_plan.py $<TARGET_FILE:npu_derive_designs_dump>)
# The host fallback GEMM reads a padded design's activations at stride a_k, inside the caller's buffer
# (engine#808 review: the split moved the engine#845 pad into the target and left the host branch on the
# raw A). The test's activation buffer carries a NaN guard tail, so an implementation that reads past the
Expand Down
7 changes: 7 additions & 0 deletions npu/forward/include/q4nx_pack.h
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,13 @@ int q4nx_derive_designs(const Q4nxModelDims* d, Q4nxDesignGeom* designs);
int q4nx_derive_decode_designs(const Q4nxDesignGeom* designs, int count,
Q4nxDesignGeom* out);

/* The UNIQUE name of a design role -- not the display tag the forward logs.
*
* They differ where a model carries two roles that read the same in a log: a qwen3_5 pack derives an
* attention QKV and a GDN QKVLIN, which the forward both prints as "QKV". A plan's `designs[].role`
* is a key (engine#803 A2), so it uses these; the log keeps the short tag. */
const char* q4nx_design_role_name(Q4nxDesign role);

/* Look up the entry for a role in a derived set; NULL if absent. */
const Q4nxDesignGeom* q4nx_find_design(const Q4nxDesignGeom* designs, int count,
Q4nxDesign role);
Expand Down
28 changes: 28 additions & 0 deletions npu/forward/src/q4nx_forward.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1088,6 +1088,34 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path,
}


// The plan may name this very set (engine#803 A2a). Checked HERE and not at the end of the
// derivation: the pb16i family appends its split designs and the b8 family rewrites every name
// with the _b8 suffix between the derivation and this loop, so this is the first point at
// which designs_[] is what a target would actually open. When it does, it has to be the set the
// container derives -- same roles, same M/K/N, same artifact -- or the run would take an ELF the
// weights were not written for. A plan without the section agrees trivially, which is what keeps
// every plan written before the field meaningful.
if (plan_.loaded()) {
std::vector<onebit::npu::plan::Plan::Design> derived;
derived.reserve((size_t)ndesigns_);
for (int i = 0; i < ndesigns_; i++) {
onebit::npu::plan::Plan::Design d;
// The plan's role is a key, so it takes the unique name; the log keeps role_tag()'s
// short tag (where QKVLIN prints as "QKV").
d.role = q4nx_design_role_name(designs_[i].role);
d.M = designs_[i].M;
d.K = designs_[i].K;
d.N = designs_[i].N;
d.elf = designs_[i].elf_name;
derived.push_back(std::move(d));
}
std::string why;
if (!plan_.designs_agree(derived, why)) {
err_ = "plan " + plan_path_ + ": " + why;
return false;
}
}

for (int i = 0; i < ndesigns_; i++) {
const Q4nxDesignGeom* g = &designs_[i];
// The two full-attention-QKV-split designs exist only as pb16i kernels (the
Expand Down
30 changes: 30 additions & 0 deletions npu/forward/src/q4nx_pack.c
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,36 @@ void q4nx_host_gemm_f32_padded(const float* A, int a_k, int M, int K, int N, con
free(pad);
}

const char* q4nx_design_role_name(Q4nxDesign role) {
switch (role) {
case Q4NX_DESIGN_QKV: return "QKV";
case Q4NX_DESIGN_QKV2: return "QKV2";
case Q4NX_DESIGN_O: return "O";
case Q4NX_DESIGN_O2: return "O2";
case Q4NX_DESIGN_GU: return "GU";
case Q4NX_DESIGN_G: return "G";
case Q4NX_DESIGN_U: return "U";
case Q4NX_DESIGN_D: return "D";
case Q4NX_DESIGN_PLEGATE: return "PLEGATE";
case Q4NX_DESIGN_PLEPROJ: return "PLEPROJ";
case Q4NX_DESIGN_GU2: return "GU2";
case Q4NX_DESIGN_D2: return "D2";
case Q4NX_DESIGN_QKVLIN: return "QKVLIN";
case Q4NX_DESIGN_Z: return "Z";
case Q4NX_DESIGN_QA: return "QA";
case Q4NX_DESIGN_QB: return "QB";
case Q4NX_DESIGN_KVA: return "KVA";
case Q4NX_DESIGN_DG: return "DG";
case Q4NX_DESIGN_DU: return "DU";
case Q4NX_DESIGN_DD: return "DD";
case Q4NX_DESIGN_GB: return "GB";
case Q4NX_DESIGN_UB: return "UB";
case Q4NX_DESIGN_QKVFA_Q: return "QKVFAQ";
case Q4NX_DESIGN_QKVFA_KV: return "QKVFAKV";
default: return "?";
}
}

/* One standard design: the model's real K and N in, the kernel's grids out, the real widths kept
* beside them. Role-specific row counts (q_rows/k_rows/v_rows, gate_rows/up_rows) are the
* caller's, because only it knows what fills N. MLA keeps its own derivation: its KVA pad is the
Expand Down
111 changes: 111 additions & 0 deletions npu/plan/plan.h
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,39 @@ struct Plan {
struct { bool enabled = false; int K = 0, N = 0; } moe_batched_gu;
std::vector<LayerPlan> layer_plans;

// One kernel the plan expects: the role it serves, the geometry it launches at and the artifact
// that implements it. `role` is the forward's own role tag ("QKV", "O", "GU", "G", "U", "D",
// "GB", "UB", the Gemma4 second attention type, the MLA and GDN roles), so a reader can match a
// design to the code that launches it without a second name table.
//
// It is the unique name (q4nx_design_role_name(): "QKV", "QKVLIN", "QA", "GB", ...), NOT the
// forward's short log tag -- those differ exactly where a model carries two roles that read alike,
// since a qwen3_5 pack derives an attention QKV and a GDN QKVLIN and both log as "QKV".
// designs_agree() still compares a multiset of (role, M, K, N, elf) rather than a role-keyed
// lookup: that is what makes a duplicated or mispaired set a refusal instead of a match, and it
// keeps the check honest if a future role name is ever reused.
//
// Why the plan carries this at all (engine#803 A2): the design set is what a target opens, and
// while it was derived in the forward, "which ELF does this role run" was answered by
// q4nx_derive_designs() plus filename probes inside one file. A plan that names the set makes
// the kernel identity part of the model's description -- and the engine then refuses a plan whose
// designs are not the ones the container derives, instead of silently running whatever the probes
// find. An absent section is the old behaviour (the probes), so every plan written before this
// field existed keeps loading.
struct Design {
std::string role;
int M = 0, K = 0, N = 0;
std::string elf;
};
std::vector<Design> designs;

// True when the plan's `designs` and a derived set name the same kernels: same roles, same M/K/N,
// same artifact, both directions (a role the plan omits is a kernel the model needs and the plan
// would not open). An empty section agrees with anything. On disagreement `err` names the role
// and both sides -- never a default, because a plan that names a different ELF than the weights
// need is exactly the silent divergence this layer exists to remove.
bool designs_agree(const std::vector<Design>& derived, std::string& err) const;

bool loaded() const { return !format.empty(); }

// Load and validate. Throws PlanError naming the field for anything this build cannot run --
Expand Down Expand Up @@ -351,6 +384,32 @@ struct Plan {
if (ternary_tag != p.pb16i)
throw PlanError("plan: " + path + " kernels.tag \"" + p.tag + "\" and container.pb16i " +
(p.pb16i ? "true" : "false") + " disagree");
// The per-role kernel set (engine#803 A2). Optional: absent means the engine keeps deriving it
// from the container's dims, which is what every plan written before this field means.
if (j.contains("designs")) {
const auto& ds = j.at("designs");
if (!ds.is_array()) throw PlanError("plan: " + path + " designs is not an array");
for (size_t i = 0; i < ds.size(); i++) {
const auto& d = ds.at(i);
const std::string where = "designs[" + std::to_string(i) + "]";
if (!d.is_object()) throw PlanError("plan: " + path + " " + where + " is not an object");
Design des;
des.role = d.value("role", "");
if (des.role.empty()) throw PlanError("plan: " + path + " " + where + ".role is empty");
// Named with the index, not just the field: a plan can carry a dozen of these and
// "M must be positive" without a position is a hunt.
des.M = d.value("M", 0);
if (des.M <= 0) throw PlanError("plan: " + path + " " + where + ".M must be positive");
des.K = d.value("K", 0);
if (des.K <= 0) throw PlanError("plan: " + path + " " + where + ".K must be positive");
des.N = d.value("N", 0);
if (des.N <= 0) throw PlanError("plan: " + path + " " + where + ".N must be positive");
des.elf = d.value("elf", "");
if (des.elf.empty())
throw PlanError("plan: " + path + " " + where + ".elf is empty (the artifact the role runs)");
p.designs.push_back(des);
}
}
return p;
}

Expand Down Expand Up @@ -397,4 +456,56 @@ struct Plan {
}
};

// Out of line for readability: the comparison is the whole point of the section. `derived` is what
// the container's own geometry produces (q4nx_derive_designs() plus the MoE pair), so this is the
// check that keeps a plan and the artifacts its weights need from drifting apart silently.
inline bool Plan::designs_agree(const std::vector<Design>& derived, std::string& err) const {
if (designs.empty()) return true; // no section: the engine derives, as it always did
// Compared as a MULTISET of (role, M, K, N, elf), not by role alone: the role tags are not unique --
// a qwen3_5 pack derives both an attention QKV and a GDN QKVLIN design, and both are logged as
// "QKV" -- so a role-keyed lookup would compare the wrong pair and, worse, report agreement.
auto same = [](const Design& a, const Design& b) {
return a.role == b.role && a.M == b.M && a.K == b.K && a.N == b.N && a.elf == b.elf;
};
std::vector<bool> used(derived.size(), false);
for (const auto& mine : designs) {
bool found = false;
for (size_t i = 0; i < derived.size(); i++) {
if (used[i] || !same(mine, derived[i])) continue;
used[i] = found = true;
break;
}
if (found) continue;
// Not found: is it a role the container does not derive at all, or the same role at a different
// geometry / artifact? The second is the dangerous one, so it says both sides.
const Design* near = nullptr;
for (const auto& d : derived)
if (d.role == mine.role) near = &d;
if (!near) {
err = "plan designs: role \"" + mine.role + "\" is not a design this container derives";
return false;
}
if (near->M != mine.M || near->K != mine.K || near->N != mine.N) {
err = "plan designs(" + mine.role + "): M=" + std::to_string(mine.M) + " K=" +
std::to_string(mine.K) + " N=" + std::to_string(mine.N) + ", the container derives M=" +
std::to_string(near->M) + " K=" + std::to_string(near->K) + " N=" +
std::to_string(near->N);
return false;
}
err = "plan designs(" + mine.role + "): elf \"" + mine.elf + "\", the container derives \"" +
near->elf + "\"";
return false;
}
// Both directions: a derived design the plan does not name is a kernel the model needs and the plan
// would not open once the target takes its set from here.
for (size_t i = 0; i < derived.size(); i++) {
if (!used[i]) {
err = "plan designs: the container derives " + derived[i].role + " (" + derived[i].elf +
") but the plan does not name it";
return false;
}
}
return true;
}

} // namespace onebit::npu::plan
78 changes: 78 additions & 0 deletions tests/npu_derive_designs_dump.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
// Copyright 2026 bong-water-water-bong
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// npu_derive_designs_dump -- print the design set the engine derives for a set of model dims.
//
// npu_derive_designs_dump H NH NKV HD IM fused_gu [attn_output_gate] [tag]
//
// One line per design: `ROLE M K N elf`, with the unique role name (q4nx_design_role_name()), the
// geometry and the artifact. It exists so the plan generator can be checked against the derivation
// instead of against a second copy of the rules: `tests/npu_plan_designs_parity_test.py` runs this
// binary and tools/npu_plan.py on the same dims and requires the two sets to be identical, which is
// the only way a plan's `designs` section (engine#803 A2a) can be trusted to name the kernels the
// container's weights were written for.
//
// No model, no device: the derivation is pure arithmetic over the dims.
#include "q4nx_pack.h"

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

int main(int argc, char** argv) {
if (argc != 7 && argc != 8 && argc != 9) {
fprintf(stderr, "usage: %s H NH NKV HD IM fused_gu [attn_output_gate] [tag]\n", argv[0]);
return 2;
}
Q4nxModelDims d;
memset(&d, 0, sizeof(d));
d.H = atoi(argv[1]);
d.NH = atoi(argv[2]);
d.NKV = atoi(argv[3]);
d.HD = atoi(argv[4]);
d.IM = atoi(argv[5]);
d.fused_gu = atoi(argv[6]);
d.attn_output_gate = argc > 7 ? atoi(argv[7]) : 0;
d.M = Q4NX_DESIGN_M_PREFILL;
/* The forward rewrites every artifact name with "_b8" for a b8 container (the blocked 8x8 tile
* layout the kernels carry); the derivation itself does not know the layout, so the dump applies
* the same rewrite when asked. Absent tag = the rm/pb16i/t2s families, whose prefill names are the
* untagged int8 ones. */
const char* tag = argc > 8 ? argv[8] : "";

Q4nxDesignGeom designs[Q4NX_MAX_DESIGNS];
memset(designs, 0, sizeof(designs));
const int n = q4nx_derive_designs(&d, designs);
if (n <= 0) {
fprintf(stderr, "derive failed for H=%d NH=%d NKV=%d HD=%d IM=%d fused_gu=%d\n", d.H, d.NH,
d.NKV, d.HD, d.IM, d.fused_gu);
return 1;
}
for (int i = 0; i < n; i++) {
char name[Q4NX_ELF_NAME_MAX];
snprintf(name, sizeof(name), "%s", designs[i].elf_name);
if (strcmp(tag, "_b8") == 0 || strcmp(tag, "b8") == 0) {
const size_t len = strlen(name);
if (len > 4 && strcmp(name + len - 4, ".elf") == 0) {
char tagged[Q4NX_ELF_NAME_MAX];
snprintf(tagged, sizeof(tagged), "%.*s_b8.elf", (int)(len - 4), name);
snprintf(name, sizeof(name), "%s", tagged);
}
}
printf("%s %d %d %d %s\n", q4nx_design_role_name(designs[i].role), designs[i].M, designs[i].K,
designs[i].N, name);
}
return 0;
}
Loading
Loading