Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions npu/pack_import.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -5287,6 +5287,17 @@ Report import_external(const std::string& pack_dir, const std::string& out_dir,
const OnnxTensor& cc = onnx.tensors[onnx.index.at("cos_cache")];
if (cc.dims.size() == 2 && cc.dims[0] > 1 && cc.dims[1] > 0 && cc.type == kF32) {
const int64_t rows = cc.dims[0], half = cc.dims[1];
// engine#2571: the cache's second dimension is rotary_dim / 2. A pack that rotates only part of each head
// (Phi-4-mini: 48 frequency columns for head_dim 128, i.e. partial_rotary_factor 0.75) and ships no config.json
// used to be served with full rotary -- coherent for a few tokens, then drifting ('multiply 12 by 7 by 7 by 7').
// Stated configs win; otherwise the cache is the pack's own statement of its rotary span.
if (!c.contains("partial_rotary_factor") && p.dims.head_dim > 0 && 2 * half < int64_t(p.dims.head_dim) &&
half > 0) {
const double prf = double(2 * half) / double(p.dims.head_dim);
c["partial_rotary_factor"] = prf;
std::fprintf(stderr, "note: partial_rotary_factor %.4g taken from the pack's cos_cache (%lld of %d head "
"dims rotate)\n", prf, (long long)(2 * half), int(p.dims.head_dim));
}
uint64_t need = 0;
if (mul_ok(uint64_t(rows) * uint64_t(half), 4u, need)) {
std::vector<float> cache(size_t(rows) * size_t(half));
Expand Down Expand Up @@ -6728,6 +6739,17 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons
const OnnxTensor& cc = onnx.tensors[onnx.index.at("cos_cache")];
if (!cc.dims.empty() && cc.dims.size() == 2 && cc.dims.at(2 % cc.dims.size()) > 0) {
const int64_t rows = cc.dims.at(0), half = cc.dims.at(1);
// engine#2571: the cache's second dimension is rotary_dim / 2. A pack that rotates only part of each head
// (Phi-4-mini: 48 frequency columns for head_dim 128, i.e. partial_rotary_factor 0.75) and ships no config.json
// used to be served with full rotary -- coherent for a few tokens, then drifting ('multiply 12 by 7 by 7 by 7').
// Stated configs win; otherwise the cache is the pack's own statement of its rotary span.
if (!c.contains("partial_rotary_factor") && p.dims.head_dim > 0 && 2 * half < int64_t(p.dims.head_dim) &&
half > 0) {
const double prf = double(2 * half) / double(p.dims.head_dim);
c["partial_rotary_factor"] = prf;
std::fprintf(stderr, "note: partial_rotary_factor %.4g taken from the pack's cos_cache (%lld of %d head "
"dims rotate)\n", prf, (long long)(2 * half), int(p.dims.head_dim));
}
const size_t n = size_t(rows) * size_t(half);
uint64_t need = 0;
if (mul_ok(n, cc.type == kF32 ? 4u : 2u, need)) {
Expand Down
19 changes: 17 additions & 2 deletions tests/npu_pack_import_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -715,15 +715,26 @@ void test_external_import_success(const std::string& root) {
std::memcpy(&final_norm[size_t(i) * 4], &v, 4);
}
const uint64_t fn_off = uint64_t(embed.size()) + uint64_t(blob.size());
put_file((dir / "eager-mm.onnx.data").string(), embed + blob + final_norm);
// engine#2571: a cos_cache with 48 frequency columns for head_dim 128 -- a pack that rotates 96 of 128 dims
// (Phi-4-mini's partial_rotary_factor 0.75) and states it nowhere else. Built at theta 10000 so the theta check passes.
const int CC_ROWS = 64, CC_HALF = 48;
std::string cos_cache(size_t(CC_ROWS) * CC_HALF * 4, '\0');
for (int pp = 0; pp < CC_ROWS; ++pp)
for (int i = 0; i < CC_HALF; ++i) {
const float v = float(std::cos(double(pp) / std::pow(10000.0, 2.0 * double(i) / double(2 * CC_HALF))));
std::memcpy(&cos_cache[(size_t(pp) * CC_HALF + i) * 4], &v, 4);
}
const uint64_t cc_off = fn_off + uint64_t(final_norm.size());
put_file((dir / "eager-mm.onnx.data").string(), embed + blob + final_norm + cos_cache);
put_file((dir / "eager-mm.onnx").string(),
pb::model({pb::tensor("model.embed_tokens.weight", {V, H}, "eager-mm.onnx.data", 0, uint64_t(embed.size())),
pb::placeholder("model.layers.0.attn.qkv_proj.MatMulNBits.qweight.empty"),
pb::placeholder("model.layers.0.attn.qkv_proj.MatMulNBits.scales.empty"),
pb::tensor("model.layers.0.attn.qkv_proj.MatMulNBits.qweight.packed", {int64_t(blob.size())},
"eager-mm.onnx.data", uint64_t(embed.size()), uint64_t(blob.size())),
pb::tensor("model.layers.1.final_norm_layernorm.weight", {H}, "eager-mm.onnx.data", fn_off,
uint64_t(final_norm.size()))},
uint64_t(final_norm.size())),
pb::tensor("cos_cache", {CC_ROWS, CC_HALF}, "eager-mm.onnx.data", cc_off, uint64_t(cos_cache.size()))},
{pb::matmulnbits("qkv", {"x", "/model/layers.0/attn/qkv_proj/MatMulNBits.qweight.empty",
"/model/layers.0/attn/qkv_proj/MatMulNBits.scales.empty", "y"},
K, N, 4, 128, "MatMulNBitsBf")}));
Expand Down Expand Up @@ -793,6 +804,10 @@ void test_external_import_success(const std::string& root) {

// config.json: the geometry, and q4nx_weight_layout ONLY for a non-default layout (engine#1840 / #1793).
const json cfg = json_of(out / "config.json");
// engine#2571: the rotary span comes from the cache's shape when the pack states it nowhere else.
check(std::fabs(cfg.value("partial_rotary_factor", 0.0) - 0.75) < 1e-9,
tag + "config.json carries partial_rotary_factor 0.75 from the 48-column cos_cache (head_dim 128)");
check(cfg.value("rope_theta", 0.0) == 10000.0, tag + "and the cache still verifies rope_theta 10000");
check(cfg.value("num_attention_heads", 0) == NH && cfg.value("num_key_value_heads", 0) == NKV &&
cfg.value("head_dim", 0) == HD && cfg.value("hidden_size", 0) == H,
tag + "config.json carries the pack's geometry");
Expand Down
Loading