diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index e479de578..589285138 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -5304,6 +5304,20 @@ Report import_external(const std::string& pack_dir, const std::string& out_dir, "keeping %.6g, attention may be wrong\n", best_corr, best_theta, double(p.dims.rope_theta)); } + // THE ROTARY SPAN, from the same cache (engine#2571): `cos_cache` carries one column per rotated + // FREQUENCY PAIR, so `2 * half` is the number of dims the pack actually rotates. Phi-4-mini rotates 96 + // of each head's 128 (HF `partial_rotary_factor: 0.75`) and its 1.7.1 hybrid packs ship **no + // config.json**, so nothing else states it; without this the forward rotated all 128 and the answer + // drifted after the first token ("multiply 12 by 7 by 7 by 7 ..."). Only written when the cache is + // narrower than the head, so a full-width cache keeps the existing behaviour. + if (p.dims.head_dim > 0 && 2 * half < p.dims.head_dim) { + const double factor = double(2 * half) / double(p.dims.head_dim); + c["partial_rotary_factor"] = factor; + std::fprintf(stderr, + "note: partial_rotary_factor %.4f (%lld of %lld head dims) from the pack's cos_cache: " + "%lld frequency columns, no config.json states the span (engine#2571)\n", + factor, (long long) (2 * half), (long long) p.dims.head_dim, (long long) half); + } } } } @@ -6759,6 +6773,20 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons if (alpha_base > 0.0) cands.insert(cands.begin(), alpha_base); double best_corr = -2.0, best_theta = p.dims.rope_theta; const int HD2 = int(half) * 2; // head_dim implied by the cache + // engine#2571: the cache states the rotary SPAN as well as the base, and this is the only place a + // pack without `config.json` says it. Phi-4-mini rotates 96 of each head's 128 dims (HF + // `partial_rotary_factor: 0.75`) and its 1.7.1 hybrid packs ship no config, so nothing wrote the + // span: the forward rotated all 128 and the answer drifted from the second token ("multiply 12 by + // 7 by 7 by 7 ...", "... on the Moon on the Moon ..."). Only when the cache is NARROWER than the + // head, so a full-width cache keeps the existing behaviour. + if (p.dims.head_dim > 0 && HD2 < p.dims.head_dim) { + const double factor = double(HD2) / double(p.dims.head_dim); + c["partial_rotary_factor"] = factor; + std::fprintf(stderr, + "note: partial_rotary_factor %.4f (%d of %lld head dims) from the pack's cos_cache: " + "%lld frequency columns, and no config states the span (engine#2571)\n", + factor, HD2, (long long) p.dims.head_dim, (long long) half); + } const int64_t pmax = std::min(rows, 4096); std::vector ang(pmax), cs(size_t(pmax)); for (double th : cands) { diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index ce384adb8..31508a4ae 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -715,7 +715,13 @@ void test_external_import_success(const std::string& root) { std::memcpy(&final_norm[size_t(i) * 4], &v, 4); } const uint64_t fn_off = uint64_t(embed.size()) + uint64_t(blob.size()); - put_file((dir / "eager-mm.onnx.data").string(), embed + blob + final_norm); + std::string cosc(size_t(64) * 48 * 4, '\0'); // engine#2571: a real-looking cache, theta 1e4 + for (int r2 = 0; r2 < 64; ++r2) + for (int i2 = 0; i2 < 48; ++i2) { + const float v = std::cos(double(r2) / std::pow(1e4, 2.0 * double(i2) / 128.0)); + std::memcpy(&cosc[size_t(r2 * 48 + i2) * 4], &v, 4); + } + put_file((dir / "eager-mm.onnx.data").string(), embed + blob + final_norm + cosc); put_file((dir / "eager-mm.onnx").string(), pb::model({pb::tensor("model.embed_tokens.weight", {V, H}, "eager-mm.onnx.data", 0, uint64_t(embed.size())), pb::placeholder("model.layers.0.attn.qkv_proj.MatMulNBits.qweight.empty"), @@ -723,7 +729,11 @@ void test_external_import_success(const std::string& root) { pb::tensor("model.layers.0.attn.qkv_proj.MatMulNBits.qweight.packed", {int64_t(blob.size())}, "eager-mm.onnx.data", uint64_t(embed.size()), uint64_t(blob.size())), pb::tensor("model.layers.1.final_norm_layernorm.weight", {H}, "eager-mm.onnx.data", fn_off, - uint64_t(final_norm.size()))}, + uint64_t(final_norm.size())), + // engine#2571: 48 frequency columns = 96 rotated dims of head_dim 128 (= HF + // partial_rotary_factor 0.75), which is all a pack without a config.json states. + pb::tensor("cos_cache", {64, 48}, "eager-mm.onnx.data", fn_off + uint64_t(final_norm.size()), + uint64_t(64 * 48 * 4))}, {pb::matmulnbits("qkv", {"x", "/model/layers.0/attn/qkv_proj/MatMulNBits.qweight.empty", "/model/layers.0/attn/qkv_proj/MatMulNBits.scales.empty", "y"}, K, N, 4, 128, "MatMulNBitsBf")})); @@ -785,6 +795,16 @@ void test_external_import_success(const std::string& root) { check(got, tag + "pseudo-layer final norm: the container carries model.norm.weight (engine#2561)"); check(got && all_match, tag + "pseudo-layer final norm: every element is the pack's, not a unit gain (engine#2561)"); + // engine#2571: the span comes from the cache's column count when no config states it. It belongs in the + // container's CONFIG (which is what q4nx_semantics.c reads), not in the tensor table -- my first version + // of this arm looked in the table and read 0 while the importer was writing 0.75. + const json cfg2 = json_of(out / "config.json"); + double prf = 0.0; + bool have = false; + if (cfg2.contains("partial_rotary_factor")) { prf = cfg2["partial_rotary_factor"].get(); have = true; } + check(have && std::fabs(prf - 0.75) < 1e-6, + tag + "partial_rotary_factor 0.75 inferred from a 48-column cos_cache (engine#2571), got " + + std::to_string(prf)); } check(why.empty() && r.refused.empty() && fs::exists(out / "model.q4nx"), tag + "a pack whose geometry the split satisfies imports to completion (" + why +