Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions npu/pack_import.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -5304,6 +5304,20 @@ Report import_external(const std::string& pack_dir, const std::string& out_dir,
"keeping %.6g, attention may be wrong\n", best_corr, best_theta,
double(p.dims.rope_theta));
}
// THE ROTARY SPAN, from the same cache (engine#2571): `cos_cache` carries one column per rotated
// FREQUENCY PAIR, so `2 * half` is the number of dims the pack actually rotates. Phi-4-mini rotates 96
// of each head's 128 (HF `partial_rotary_factor: 0.75`) and its 1.7.1 hybrid packs ship **no
// config.json**, so nothing else states it; without this the forward rotated all 128 and the answer
// drifted after the first token ("multiply 12 by 7 by 7 by 7 ..."). Only written when the cache is
// narrower than the head, so a full-width cache keeps the existing behaviour.
if (p.dims.head_dim > 0 && 2 * half < p.dims.head_dim) {
const double factor = double(2 * half) / double(p.dims.head_dim);
c["partial_rotary_factor"] = factor;
std::fprintf(stderr,
"note: partial_rotary_factor %.4f (%lld of %lld head dims) from the pack's cos_cache: "
"%lld frequency columns, no config.json states the span (engine#2571)\n",
factor, (long long) (2 * half), (long long) p.dims.head_dim, (long long) half);
}
}
}
}
Expand Down Expand Up @@ -6759,6 +6773,20 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons
if (alpha_base > 0.0) cands.insert(cands.begin(), alpha_base);
double best_corr = -2.0, best_theta = p.dims.rope_theta;
const int HD2 = int(half) * 2; // head_dim implied by the cache
// engine#2571: the cache states the rotary SPAN as well as the base, and this is the only place a
// pack without `config.json` says it. Phi-4-mini rotates 96 of each head's 128 dims (HF
// `partial_rotary_factor: 0.75`) and its 1.7.1 hybrid packs ship no config, so nothing wrote the
// span: the forward rotated all 128 and the answer drifted from the second token ("multiply 12 by
// 7 by 7 by 7 ...", "... on the Moon on the Moon ..."). Only when the cache is NARROWER than the
// head, so a full-width cache keeps the existing behaviour.
if (p.dims.head_dim > 0 && HD2 < p.dims.head_dim) {
const double factor = double(HD2) / double(p.dims.head_dim);
c["partial_rotary_factor"] = factor;
std::fprintf(stderr,
"note: partial_rotary_factor %.4f (%d of %lld head dims) from the pack's cos_cache: "
"%lld frequency columns, and no config states the span (engine#2571)\n",
factor, HD2, (long long) p.dims.head_dim, (long long) half);
}
const int64_t pmax = std::min<int64_t>(rows, 4096);
std::vector<double> ang(pmax), cs(size_t(pmax));
for (double th : cands) {
Expand Down
24 changes: 22 additions & 2 deletions tests/npu_pack_import_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -715,15 +715,25 @@ void test_external_import_success(const std::string& root) {
std::memcpy(&final_norm[size_t(i) * 4], &v, 4);
}
const uint64_t fn_off = uint64_t(embed.size()) + uint64_t(blob.size());
put_file((dir / "eager-mm.onnx.data").string(), embed + blob + final_norm);
std::string cosc(size_t(64) * 48 * 4, '\0'); // engine#2571: a real-looking cache, theta 1e4
for (int r2 = 0; r2 < 64; ++r2)
for (int i2 = 0; i2 < 48; ++i2) {
const float v = std::cos(double(r2) / std::pow(1e4, 2.0 * double(i2) / 128.0));
std::memcpy(&cosc[size_t(r2 * 48 + i2) * 4], &v, 4);
}
put_file((dir / "eager-mm.onnx.data").string(), embed + blob + final_norm + cosc);
put_file((dir / "eager-mm.onnx").string(),
pb::model({pb::tensor("model.embed_tokens.weight", {V, H}, "eager-mm.onnx.data", 0, uint64_t(embed.size())),
pb::placeholder("model.layers.0.attn.qkv_proj.MatMulNBits.qweight.empty"),
pb::placeholder("model.layers.0.attn.qkv_proj.MatMulNBits.scales.empty"),
pb::tensor("model.layers.0.attn.qkv_proj.MatMulNBits.qweight.packed", {int64_t(blob.size())},
"eager-mm.onnx.data", uint64_t(embed.size()), uint64_t(blob.size())),
pb::tensor("model.layers.1.final_norm_layernorm.weight", {H}, "eager-mm.onnx.data", fn_off,
uint64_t(final_norm.size()))},
uint64_t(final_norm.size())),
// engine#2571: 48 frequency columns = 96 rotated dims of head_dim 128 (= HF
// partial_rotary_factor 0.75), which is all a pack without a config.json states.
pb::tensor("cos_cache", {64, 48}, "eager-mm.onnx.data", fn_off + uint64_t(final_norm.size()),
uint64_t(64 * 48 * 4))},
{pb::matmulnbits("qkv", {"x", "/model/layers.0/attn/qkv_proj/MatMulNBits.qweight.empty",
"/model/layers.0/attn/qkv_proj/MatMulNBits.scales.empty", "y"},
K, N, 4, 128, "MatMulNBitsBf")}));
Expand Down Expand Up @@ -785,6 +795,16 @@ void test_external_import_success(const std::string& root) {
check(got, tag + "pseudo-layer final norm: the container carries model.norm.weight (engine#2561)");
check(got && all_match,
tag + "pseudo-layer final norm: every element is the pack's, not a unit gain (engine#2561)");
// engine#2571: the span comes from the cache's column count when no config states it. It belongs in the
// container's CONFIG (which is what q4nx_semantics.c reads), not in the tensor table -- my first version
// of this arm looked in the table and read 0 while the importer was writing 0.75.
const json cfg2 = json_of(out / "config.json");
double prf = 0.0;
bool have = false;
if (cfg2.contains("partial_rotary_factor")) { prf = cfg2["partial_rotary_factor"].get<double>(); have = true; }
check(have && std::fabs(prf - 0.75) < 1e-6,
tag + "partial_rotary_factor 0.75 inferred from a 48-column cos_cache (engine#2571), got " +
std::to_string(prf));
}
check(why.empty() && r.refused.empty() && fs::exists(out / "model.q4nx"),
tag + "a pack whose geometry the split satisfies imports to completion (" + why +
Expand Down
Loading