diff --git a/CMakeLists.txt b/CMakeLists.txt index 0f34f44b4..e73fafab1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1614,6 +1614,12 @@ add_test(NAME batch_check_compare COMMAND batch_check_compare_test) # engine#1780: the DFlash draft block's acceptance arithmetic -- tokens = accepted + bonus, the break-even rule, and the # published OGA-route counts as known answers. Header-only and host-only: no container, no device, no NPU runtime, so it # runs wherever this file is built. Registered at top level with the same hook as the row above, for the same reason. +# The per-layer weight-cache keys are distinct (npu/forward/include/weight_keys.h): two projections of one layer +# under one key launch with the same weights on the device. Header-only, no model. +add_executable(weight_keys_test tests/weight_keys_test.cpp) +target_include_directories(weight_keys_test PRIVATE npu/forward/include) +add_test(NAME weight_keys COMMAND weight_keys_test) +add_dependencies(onebit weight_keys_test) # engine#1780: where a tapped layer's rows go (npu/forward/include/layer_taps.h), header-only, no model. add_executable(layer_taps_test tests/layer_taps_test.cpp) target_include_directories(layer_taps_test PRIVATE npu/forward/include) diff --git a/docs/npu.md b/docs/npu.md index e1bb246f1..ce6e200af 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -2889,6 +2889,14 @@ can fail a run. variable and never sees that line is running a binary built before the dump existed (#1977): rebuild it, rather than looking for a code path that skips the dump. +**A projection that is right on the host and wrong on the device: check its weight-cache key.** The device target +keeps one packed weight image per (layer, key), and the host arms do not use that cache. Two projections of one layer +under one key therefore launch with the same weights on the NPU only: a Gemma4 pack answered correctly in host-GEMM +mode and returned noise on the device, with every kernel bit-exact, because its per-layer input gate and projection +used the attention projections' keys (engine#2213). `NPU_INFER_DUMP_OPS=1` on both sides showed it in one token: the +op lines agree up to the first projection that shares a key. A layer's keys are named in +`npu/forward/include/weight_keys.h` and held distinct at compile time; a new projection takes a new name there. + A caller inside the process that needs the same stream as data, not files, uses `Q4nxNpuForward::set_layer_taps(layers, &sink)` (engine#1780): every pass then appends one row per position to the sink, `taps x H` wide, the listed layers side by side. It works for `step()` and for the batched passes, and it is what diff --git a/npu/forward/include/weight_keys.h b/npu/forward/include/weight_keys.h new file mode 100644 index 000000000..9f6080a67 --- /dev/null +++ b/npu/forward/include/weight_keys.h @@ -0,0 +1,65 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// weight_keys.h -- the keys a layer's projections are cached under on the device. +// +// The device target keeps one packed weight image per (layer, key) (XdnaTarget's weight cache), so two projections +// of the same layer that share a key share an image: the second one launches with the first one's weights. The host +// arms do not go through that cache, so a collision is invisible off the device -- a Gemma4 pack answered correctly +// in host-GEMM mode and returned noise on the NPU, with every kernel bit-exact, because its per-layer input gate and +// projection were launched under the attention projections' keys 0 and 1. +// +// The keys of step()'s dense layer are listed here and held pairwise distinct at compile time. +// +// Which call sites use the names today: only the two that were wrong, the Gemma4 gate and projection in step() +// (kWkeyPleGate, kWkeyPleProj). The older call sites still pass literals -- 0..4 and 10..13 in step() and in the +// batched body -- and the names above carry the same numbers, so the list describes them but does not yet bind +// them. Moving those to the names is a mechanical follow-up. Until then the rule for a new projection is: take a +// new name here, not a number there. + +#pragma once + +namespace onebit::npu { + +enum : int { + kWkeyQkv = 0, // attention q|k|v (or the first half of a split design) + kWkeyO = 1, // attention output + kWkeyGateUp = 2, // fused gate|up, or gate alone on a split MLP + kWkeyUp = 3, // up alone on a split MLP + kWkeyDown = 4, // MLP down + kWkeyPleGate = 8, // Gemma4 per-layer input gate + kWkeyPleProj = 9, // Gemma4 per-layer projection + kWkeySplitQ0 = 10, // split full-attention halves + kWkeySplitQ1 = 11, + kWkeySplitK = 12, + kWkeySplitV = 13, +}; + +constexpr int kLayerWkeys[] = {kWkeyQkv, kWkeyO, kWkeyGateUp, kWkeyUp, kWkeyDown, kWkeyPleGate, kWkeyPleProj, + kWkeySplitQ0, kWkeySplitQ1, kWkeySplitK, kWkeySplitV}; + +// The predicate on any list, so a test can hand it one that is NOT distinct and see it say so. +constexpr bool wkeys_distinct(const int* keys, int n) { + for (int i = 0; i < n; i++) + for (int j = i + 1; j < n; j++) + if (keys[i] == keys[j]) return false; + return true; +} +constexpr bool layer_wkeys_distinct() { + return wkeys_distinct(kLayerWkeys, int(sizeof(kLayerWkeys) / sizeof(kLayerWkeys[0]))); +} +static_assert(layer_wkeys_distinct(), "two projections of one layer share a weight-cache key"); + +} // namespace onebit::npu diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 7a45823e1..c55d73522 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -24,6 +24,7 @@ // Numerics follow tools/q4nx_forward_ref.py so the two can be compared // element-wise. #include "q4nx_forward.h" +#include "weight_keys.h" // the per-layer weight-cache keys, held distinct at compile time #include "layer_taps.h" // engine#1780: where a tapped layer's rows go in the sink #include "pad_head_rows.h" // engine#1990: the padded-head row rule, header-only for its arm #include "batch_width.h" @@ -5420,8 +5421,10 @@ bool Q4nxNpuForward::step_impl(int token, int pos, std::vector& logits, i const Q4nxDesignGeom* g = &designs_[i_pg]; Q4nxProjections p{}; p.ple = inp_gate_[l].data(); - if (!weight_cached(0) && q4nx_assemble_B(g, &p, B_.data()) != 0) { err_ = "assemble plegate"; return false; } - npu_gemm(i_pg, x.data(), g->K, g->N, B_.data(), pg, 0, nullptr, g->real_k); + // The layer's own weight-cache keys (weight_keys.h): 0 and 1 are this layer's QKV and O, and a gate + // launched under key 0 ran with the QKV image on the device -- the host arms have no such cache. + if (!weight_cached(onebit::npu::kWkeyPleGate) && q4nx_assemble_B(g, &p, B_.data()) != 0) { err_ = "assemble plegate"; return false; } + npu_gemm(i_pg, x.data(), g->K, g->N, B_.data(), pg, onebit::npu::kWkeyPleGate, "ple_gate", g->real_k); } const int gelu = (sem_.mlp_act == Q4NX_ACT_GELU); for (int i = 0; i < ple_dim_; i++) { @@ -5439,8 +5442,8 @@ bool Q4nxNpuForward::step_impl(int token, int pos, std::vector& logits, i const Q4nxDesignGeom* g = &designs_[i_pp]; Q4nxProjections p{}; p.ple = ple_proj_[l].data(); - if (!weight_cached(1) && q4nx_assemble_B(g, &p, B_.data()) != 0) { err_ = "assemble pleproj"; return false; } - npu_gemm(i_pp, pg.data(), g->K, g->N, B_.data(), pp, 1, nullptr, g->real_k); + if (!weight_cached(onebit::npu::kWkeyPleProj) && q4nx_assemble_B(g, &p, B_.data()) != 0) { err_ = "assemble pleproj"; return false; } + npu_gemm(i_pp, pg.data(), g->K, g->N, B_.data(), pp, onebit::npu::kWkeyPleProj, "ple_proj", g->real_k); } rmsnorm(pp.data(), post_ln_[l].data(), H, xn.data()); for (int k = 0; k < H; k++) x[k] += xn[k]; diff --git a/tests/weight_keys_test.cpp b/tests/weight_keys_test.cpp new file mode 100644 index 000000000..604a862e9 --- /dev/null +++ b/tests/weight_keys_test.cpp @@ -0,0 +1,51 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// weight_keys_test.cpp -- the per-layer weight-cache keys (npu/forward/include/weight_keys.h) are pairwise distinct, +// and the two Gemma4 per-layer-input projections do not use the attention projections' keys. The header asserts the +// first at compile time; this test is what makes CI compile it, and it names the pair that broke. + +#include + +#include "weight_keys.h" + +int main() { + using namespace onebit::npu; + int fail = 0; + const int n = int(sizeof(kLayerWkeys) / sizeof(kLayerWkeys[0])); + for (int i = 0; i < n; i++) + for (int j = i + 1; j < n; j++) + if (kLayerWkeys[i] == kLayerWkeys[j]) { + std::printf("FAIL keys %d and %d of the list are both %d\n", i, j, kLayerWkeys[i]); + fail++; + } + if (kWkeyPleGate == kWkeyQkv || kWkeyPleGate == kWkeyO || kWkeyPleProj == kWkeyQkv || kWkeyPleProj == kWkeyO) { + std::printf("FAIL a per-layer-input projection shares a key with an attention projection of the same layer\n"); + fail++; + } + // The predicate is not a constant: a list with a repeat is reported, wherever the repeat sits. + const int dup_ends[] = {0, 1, 2, 0}, dup_mid[] = {0, 8, 8, 9}, one[] = {5}; + if (wkeys_distinct(dup_ends, 4) || wkeys_distinct(dup_mid, 4)) { + std::printf("FAIL a list with a repeated key is called distinct\n"); + fail++; + } + if (!wkeys_distinct(one, 1) || !wkeys_distinct(one, 0)) { + std::printf("FAIL a list of one key, or of none, is called not distinct\n"); + fail++; + } + if (fail) return 1; + std::printf("weight_keys: ok (%d keys, all distinct; a repeated key is reported)\n", n); + return 0; +}