From 039cf756e326f012378cba653ef13889993ddff6 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 15:14:03 -0400 Subject: [PATCH] npu-pack: a pack without config.json takes its rotary span from the cos_cache Phi-4-mini rotates 96 of each head's 128 dims (HF partial_rotary_factor 0.75) and its 1.7.1 hybrid packs ship no config.json, so nothing stated the span: the importer wrote none, the forward rotated all 128, and the answer drifted from the second token ("multiply 12 by 7 by 7 by 7 ...", "... on the Moon on the Moon ...") with a passing head self-check (corr 0.9963), which is why it read as a kernel problem for a while. The pack does state it: the attention's cos_cache has one column per rotated FREQUENCY PAIR, so 2 * columns is the rotated width. Both writers already read that cache to verify rope_theta and discard the span; they now keep it, as a top-level partial_rotary_factor (the key q4nx_semantics.c:402 reads), and only when the cache is NARROWER than the head -- a full-width cache is left exactly as before. Arm: the external-success fixture gains a 48-column cos_cache beside the pseudo-layer norm and asserts the container's config says 0.75. Measured both ways on npu_pack_import_test: fix stashed 2 FAIL (partial_rotary_factor 0.75 inferred from a 48-column cos_cache, got 0) this commit 0 FAIL, rc 0 --- npu/pack_import.cpp | 28 ++++++++++++++++++++++++++++ tests/npu_pack_import_test.cpp | 24 ++++++++++++++++++++++-- 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index e479de578..589285138 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -5304,6 +5304,20 @@ Report import_external(const std::string& pack_dir, const std::string& out_dir, "keeping %.6g, attention may be wrong\n", best_corr, best_theta, double(p.dims.rope_theta)); } + // THE ROTARY SPAN, from the same cache (engine#2571): `cos_cache` carries one column per rotated + // FREQUENCY PAIR, so `2 * half` is the number of dims the pack actually rotates. Phi-4-mini rotates 96 + // of each head's 128 (HF `partial_rotary_factor: 0.75`) and its 1.7.1 hybrid packs ship **no + // config.json**, so nothing else states it; without this the forward rotated all 128 and the answer + // drifted after the first token ("multiply 12 by 7 by 7 by 7 ..."). Only written when the cache is + // narrower than the head, so a full-width cache keeps the existing behaviour. + if (p.dims.head_dim > 0 && 2 * half < p.dims.head_dim) { + const double factor = double(2 * half) / double(p.dims.head_dim); + c["partial_rotary_factor"] = factor; + std::fprintf(stderr, + "note: partial_rotary_factor %.4f (%lld of %lld head dims) from the pack's cos_cache: " + "%lld frequency columns, no config.json states the span (engine#2571)\n", + factor, (long long) (2 * half), (long long) p.dims.head_dim, (long long) half); + } } } } @@ -6759,6 +6773,20 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons if (alpha_base > 0.0) cands.insert(cands.begin(), alpha_base); double best_corr = -2.0, best_theta = p.dims.rope_theta; const int HD2 = int(half) * 2; // head_dim implied by the cache + // engine#2571: the cache states the rotary SPAN as well as the base, and this is the only place a + // pack without `config.json` says it. Phi-4-mini rotates 96 of each head's 128 dims (HF + // `partial_rotary_factor: 0.75`) and its 1.7.1 hybrid packs ship no config, so nothing wrote the + // span: the forward rotated all 128 and the answer drifted from the second token ("multiply 12 by + // 7 by 7 by 7 ...", "... on the Moon on the Moon ..."). Only when the cache is NARROWER than the + // head, so a full-width cache keeps the existing behaviour. + if (p.dims.head_dim > 0 && HD2 < p.dims.head_dim) { + const double factor = double(HD2) / double(p.dims.head_dim); + c["partial_rotary_factor"] = factor; + std::fprintf(stderr, + "note: partial_rotary_factor %.4f (%d of %lld head dims) from the pack's cos_cache: " + "%lld frequency columns, and no config states the span (engine#2571)\n", + factor, HD2, (long long) p.dims.head_dim, (long long) half); + } const int64_t pmax = std::min(rows, 4096); std::vector ang(pmax), cs(size_t(pmax)); for (double th : cands) { diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index ce384adb8..31508a4ae 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -715,7 +715,13 @@ void test_external_import_success(const std::string& root) { std::memcpy(&final_norm[size_t(i) * 4], &v, 4); } const uint64_t fn_off = uint64_t(embed.size()) + uint64_t(blob.size()); - put_file((dir / "eager-mm.onnx.data").string(), embed + blob + final_norm); + std::string cosc(size_t(64) * 48 * 4, '\0'); // engine#2571: a real-looking cache, theta 1e4 + for (int r2 = 0; r2 < 64; ++r2) + for (int i2 = 0; i2 < 48; ++i2) { + const float v = std::cos(double(r2) / std::pow(1e4, 2.0 * double(i2) / 128.0)); + std::memcpy(&cosc[size_t(r2 * 48 + i2) * 4], &v, 4); + } + put_file((dir / "eager-mm.onnx.data").string(), embed + blob + final_norm + cosc); put_file((dir / "eager-mm.onnx").string(), pb::model({pb::tensor("model.embed_tokens.weight", {V, H}, "eager-mm.onnx.data", 0, uint64_t(embed.size())), pb::placeholder("model.layers.0.attn.qkv_proj.MatMulNBits.qweight.empty"), @@ -723,7 +729,11 @@ void test_external_import_success(const std::string& root) { pb::tensor("model.layers.0.attn.qkv_proj.MatMulNBits.qweight.packed", {int64_t(blob.size())}, "eager-mm.onnx.data", uint64_t(embed.size()), uint64_t(blob.size())), pb::tensor("model.layers.1.final_norm_layernorm.weight", {H}, "eager-mm.onnx.data", fn_off, - uint64_t(final_norm.size()))}, + uint64_t(final_norm.size())), + // engine#2571: 48 frequency columns = 96 rotated dims of head_dim 128 (= HF + // partial_rotary_factor 0.75), which is all a pack without a config.json states. + pb::tensor("cos_cache", {64, 48}, "eager-mm.onnx.data", fn_off + uint64_t(final_norm.size()), + uint64_t(64 * 48 * 4))}, {pb::matmulnbits("qkv", {"x", "/model/layers.0/attn/qkv_proj/MatMulNBits.qweight.empty", "/model/layers.0/attn/qkv_proj/MatMulNBits.scales.empty", "y"}, K, N, 4, 128, "MatMulNBitsBf")})); @@ -785,6 +795,16 @@ void test_external_import_success(const std::string& root) { check(got, tag + "pseudo-layer final norm: the container carries model.norm.weight (engine#2561)"); check(got && all_match, tag + "pseudo-layer final norm: every element is the pack's, not a unit gain (engine#2561)"); + // engine#2571: the span comes from the cache's column count when no config states it. It belongs in the + // container's CONFIG (which is what q4nx_semantics.c reads), not in the tensor table -- my first version + // of this arm looked in the table and read 0 while the importer was writing 0.75. + const json cfg2 = json_of(out / "config.json"); + double prf = 0.0; + bool have = false; + if (cfg2.contains("partial_rotary_factor")) { prf = cfg2["partial_rotary_factor"].get(); have = true; } + check(have && std::fabs(prf - 0.75) < 1e-6, + tag + "partial_rotary_factor 0.75 inferred from a 48-column cos_cache (engine#2571), got " + + std::to_string(prf)); } check(why.empty() && r.refused.empty() && fs::exists(out / "model.q4nx"), tag + "a pack whose geometry the split satisfies imports to completion (" + why +