Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions npu/forward/src/q4nx_forward.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1011,6 +1011,28 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path,
fprintf(stderr, " [forward] derived %d designs (fused_gu=%d, H=%d NH=%d NKV=%d HD=%d IM=%d)\n",
ndesigns_, dims.fused_gu, dims.H, dims.NH, dims.NKV, dims.HD, dims.IM);

// The container states which design family it was imported for, and a serve that derives a different one refuses
// rather than answering garbage. Two containers with the same hidden size can differ in IM, head count or head dim
// (so in every K/N the kernels are looked up by), and pointing one at the other's ELF set used to be a silent
// wrong answer. The field is written by `1bit npu-pack import` as "ROLE:KxN,ROLE:KxN,..."; a container without it
// (everything imported before this) is unaffected.
{
const std::string want = read_config_string(cfg_path, "\"q4nx_designs\"", "");
if (!want.empty()) {
std::string have;
for (int i = 0; i < ndesigns_; i++) {
have += (have.empty() ? "" : ",") + std::string(role_tag(designs_[i].role)) + ":" +
std::to_string(designs_[i].K) + "x" + std::to_string(designs_[i].N);
}
if (have != want) {
err_ = "this container declares its kernel family as \"" + want + "\", but this model derives \"" +
have + "\": the ELFs it was imported for are not the ones this geometry needs, so it would "
"answer wrong text rather than fail (re-import, or serve the container that matches)";
return false;
}
}
}

// pb16i has no kernel for a wide fused full-attention QKV: its q half needs the
// int64 accumulator, and 14336/32 = 448 int64 columns per core do not fit L1. The
// role therefore runs as the two q halves (N = NH*HD each, i.e. q and the output
Expand Down
35 changes: 35 additions & 0 deletions npu/pack_import.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -518,6 +518,17 @@ json load_geometry(const std::string& pack_dir, std::string* source) {
if (m.contains("vocab_size")) cfg["vocab_size"] = m["vocab_size"];
if (m.contains("context_length")) cfg["max_position_embeddings"] = m["context_length"];
if (m.contains("type") && m["type"].is_string()) cfg["model_type"] = m["type"];
// The stop/start tokens live HERE on this line -- these packs ship no generation_config.json at all, which I
// had assumed they did. A container that does not state its eos cannot stop on it.
for (const char* k : {"eos_token_id", "bos_token_id", "pad_token_id"})
if (m.contains(k) && !m[k].is_null()) cfg[k] = m[k];
if (m.contains("type") && m["type"].is_string()) {
// genai_config states the family in lower case ("qwen2", "llama"); the Hugging Face spelling that other
// routes match is capitalised, so it is normalised rather than concatenated as-is.
std::string fam = m["type"].get<std::string>();
if (!fam.empty()) fam[0] = char(std::toupper((unsigned char)fam[0]));
cfg["architectures"] = std::vector<std::string>{fam + "ForCausalLM"};
}
if (source) *source = "genai_config.json (decoder block)";
return cfg;
}
Expand Down Expand Up @@ -2336,6 +2347,30 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons
}
}
c["rms_norm_eps"] = p.dims.rms_norm_eps;
// The kernel family this container was imported for, so a serve that derives a different one refuses instead of
// answering garbage (the forward compares this exact string). Shape per role, in the order the importer plans
// them: QKV is the fused q|k|v width, GU the fused gate|up width, both on the padded K grid.
{
// The forward's own grids (npu/forward/include/q4nx_pack.h): K pads to the 256-column grid, N to the
// 512-column one. Spelled out here because the importer is a different build target from that helper;
// the forward compares this exact string, so a drift between the two shows up as a refusal (loud)
// rather than as wrong text (silent). Verified against both packs' `derived 4 designs` lines:
// SmolLM2 QKV 768x1024 / O 768x1024 / GU 768x3072 / D 1536x1024; Qwen2.5-0.5B 1024x1536 / 1024x1024 /
// 1024x9728 / 4864x1024.
auto pad_k = [](int k) { return (k + 255) / 256 * 256; };
auto pad_n = [](int n) { return (n + 511) / 512 * 512; };
const int kpad = pad_k(p.dims.hidden);
const int q_out = p.dims.heads * p.dims.head_dim, kv_out = p.dims.kv_heads * p.dims.head_dim;
c["q4nx_designs"] = "QKV:" + std::to_string(kpad) + "x" + std::to_string(pad_n(q_out + 2 * kv_out)) +
",O:" + std::to_string(kpad) + "x" + std::to_string(pad_n(p.dims.hidden)) +
",GU:" + std::to_string(kpad) + "x" + std::to_string(pad_n(2 * p.dims.intermediate)) +
",D:" + std::to_string(pad_k(p.dims.intermediate)) + "x" +
std::to_string(pad_n(p.dims.hidden));
}
// The stop tokens: `load_geometry` already copied eos/bos/pad and `architectures` out of the pack's
// genai_config.json (these packs ship NO generation_config.json -- I had assumed they did), and read_dims
// parsed eos into p.dims. This line is the belt: whatever the plan resolved wins over the raw metadata.
if (!p.dims.eos.empty()) c["eos_token_id"] = p.dims.eos;
// ── the RoPE base, verified against the pack's own cache ──
// These packs ship precomputed cos_cache/sin_cache, and their genai_config.json is not a trustworthy source
// for theta (pi-inferzo measured 10000 declared where the model needs 1e6). Our forward COMPUTES RoPE from
Expand Down
Loading