From 236a88d999c4597020c11b3a182d52eecb9c6e5f Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 8 Nov 2026 09:29:27 -0400 Subject: [PATCH 1/2] npu-pack: a genre-A container states the pack's stop tokens and its architecture These packs ship no generation_config.json at all -- I had assumed they did, and the gap only showed up in a config diff against the 1.8 container of the same model, which carries eos_token_id, bos_token_id, architectures and friends. The container is what a serve reads months later, so it should state them rather than relying on a file the pack does not have. load_geometry now carries eos_token_id / bos_token_id / pad_token_id and `architectures` out of the pack's genai_config.json's model block (normalising the family name to the Hugging Face spelling -- genai_config says "qwen2", runners match "Qwen2"), and the container writer keeps the plan's resolved eos when it has one. Measured on both hybrid packs: Qwen2.5-0.5B rai_1.7.1: eos_token_id [151645, 151643], bos 151643, pad 151643, architectures ["Qwen2ForCausalLM"], import unchanged at 291 tensors / 67,088 tiles / self-check 0.994548 SmolLM2-135M rai_1.7.1: eos_token_id [2], architectures ["LlamaForCausalLM"], import unchanged at 363 tensors / 20,808 tiles / self-check 0.995685, and the text arm still answers "The capital of France is" -> " Paris. Paris is the largest city in the world" ctest npu_pack_import + attn_math pass --- npu/pack_import.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 9286a7412..56d4e5bde 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -518,6 +518,17 @@ json load_geometry(const std::string& pack_dir, std::string* source) { if (m.contains("vocab_size")) cfg["vocab_size"] = m["vocab_size"]; if (m.contains("context_length")) cfg["max_position_embeddings"] = m["context_length"]; if (m.contains("type") && m["type"].is_string()) cfg["model_type"] = m["type"]; + // The stop/start tokens live HERE on this line -- these packs ship no generation_config.json at all, which I + // had assumed they did. A container that does not state its eos cannot stop on it. + for (const char* k : {"eos_token_id", "bos_token_id", "pad_token_id"}) + if (m.contains(k) && !m[k].is_null()) cfg[k] = m[k]; + if (m.contains("type") && m["type"].is_string()) { + // genai_config states the family in lower case ("qwen2", "llama"); the Hugging Face spelling that other + // routes match is capitalised, so it is normalised rather than concatenated as-is. + std::string fam = m["type"].get(); + if (!fam.empty()) fam[0] = char(std::toupper((unsigned char)fam[0])); + cfg["architectures"] = std::vector{fam + "ForCausalLM"}; + } if (source) *source = "genai_config.json (decoder block)"; return cfg; } @@ -2336,6 +2347,10 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons } } c["rms_norm_eps"] = p.dims.rms_norm_eps; + // The stop tokens: `load_geometry` already copied eos/bos/pad and `architectures` out of the pack's + // genai_config.json (these packs ship NO generation_config.json -- I had assumed they did), and read_dims + // parsed eos into p.dims. This line is the belt: whatever the plan resolved wins over the raw metadata. + if (!p.dims.eos.empty()) c["eos_token_id"] = p.dims.eos; // ── the RoPE base, verified against the pack's own cache ── // These packs ship precomputed cos_cache/sin_cache, and their genai_config.json is not a trustworthy source // for theta (pi-inferzo measured 10000 declared where the model needs 1e6). Our forward COMPUTES RoPE from From dbbfd775c2f685795f669bcb77049284121f2187 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 8 Nov 2026 09:47:53 -0400 Subject: [PATCH 2/2] npu-pack: a container states the kernel family it was imported for, and a serve that derives another one refuses pi-inferzo's request, and the failure mode behind it is real: two containers can share a hidden size and differ in intermediate width, head count or head dim, i.e. in every K/N the kernels are looked up by, so serving one against the other's ELF set used to be a silent wrong answer. Import now writes the family into the container, and the forward compares it against what it derives: "q4nx_designs": "QKV:x,O:x,GU:x,D:x" with K on the forward's 256-column grid and N on its 512-column one (npu/forward/include/q4nx_pack.h), because that is what the loader derives and looks up -- an unpadded string would have refused every correct container. Measured against both packs' own "derived 4 designs" lines: SmolLM2-135M QKV 768x1024 / O 768x1024 / GU 768x3072 / D 1536x1024, and Qwen2.5-0.5B QKV 1024x1536 / O 1024x1024 / GU 1024x9728 / D 4864x1024 -- both match to the number. Verified both directions, device-free: matching container -> serves same container with the other family's string -> refuses at init: "this container declares its kernel family as \"QKV:1024x1536,...\", but this model derives \"QKV:768x1024,...\": the ELFs it was imported for are not the ones this geometry needs, so it would answer wrong text rather than fail (re-import, or serve the container that matches)" A container without the key (everything imported before this) is unaffected, so no existing directory starts refusing. ctest npu_pack_import + attn_math pass. --- npu/forward/src/q4nx_forward.cpp | 22 ++++++++++++++++++++++ npu/pack_import.cpp | 20 ++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index fd3217911..851a459a6 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -1011,6 +1011,28 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, fprintf(stderr, " [forward] derived %d designs (fused_gu=%d, H=%d NH=%d NKV=%d HD=%d IM=%d)\n", ndesigns_, dims.fused_gu, dims.H, dims.NH, dims.NKV, dims.HD, dims.IM); + // The container states which design family it was imported for, and a serve that derives a different one refuses + // rather than answering garbage. Two containers with the same hidden size can differ in IM, head count or head dim + // (so in every K/N the kernels are looked up by), and pointing one at the other's ELF set used to be a silent + // wrong answer. The field is written by `1bit npu-pack import` as "ROLE:KxN,ROLE:KxN,..."; a container without it + // (everything imported before this) is unaffected. + { + const std::string want = read_config_string(cfg_path, "\"q4nx_designs\"", ""); + if (!want.empty()) { + std::string have; + for (int i = 0; i < ndesigns_; i++) { + have += (have.empty() ? "" : ",") + std::string(role_tag(designs_[i].role)) + ":" + + std::to_string(designs_[i].K) + "x" + std::to_string(designs_[i].N); + } + if (have != want) { + err_ = "this container declares its kernel family as \"" + want + "\", but this model derives \"" + + have + "\": the ELFs it was imported for are not the ones this geometry needs, so it would " + "answer wrong text rather than fail (re-import, or serve the container that matches)"; + return false; + } + } + } + // pb16i has no kernel for a wide fused full-attention QKV: its q half needs the // int64 accumulator, and 14336/32 = 448 int64 columns per core do not fit L1. The // role therefore runs as the two q halves (N = NH*HD each, i.e. q and the output diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 56d4e5bde..d2adf33cf 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -2347,6 +2347,26 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons } } c["rms_norm_eps"] = p.dims.rms_norm_eps; + // The kernel family this container was imported for, so a serve that derives a different one refuses instead of + // answering garbage (the forward compares this exact string). Shape per role, in the order the importer plans + // them: QKV is the fused q|k|v width, GU the fused gate|up width, both on the padded K grid. + { + // The forward's own grids (npu/forward/include/q4nx_pack.h): K pads to the 256-column grid, N to the + // 512-column one. Spelled out here because the importer is a different build target from that helper; + // the forward compares this exact string, so a drift between the two shows up as a refusal (loud) + // rather than as wrong text (silent). Verified against both packs' `derived 4 designs` lines: + // SmolLM2 QKV 768x1024 / O 768x1024 / GU 768x3072 / D 1536x1024; Qwen2.5-0.5B 1024x1536 / 1024x1024 / + // 1024x9728 / 4864x1024. + auto pad_k = [](int k) { return (k + 255) / 256 * 256; }; + auto pad_n = [](int n) { return (n + 511) / 512 * 512; }; + const int kpad = pad_k(p.dims.hidden); + const int q_out = p.dims.heads * p.dims.head_dim, kv_out = p.dims.kv_heads * p.dims.head_dim; + c["q4nx_designs"] = "QKV:" + std::to_string(kpad) + "x" + std::to_string(pad_n(q_out + 2 * kv_out)) + + ",O:" + std::to_string(kpad) + "x" + std::to_string(pad_n(p.dims.hidden)) + + ",GU:" + std::to_string(kpad) + "x" + std::to_string(pad_n(2 * p.dims.intermediate)) + + ",D:" + std::to_string(pad_k(p.dims.intermediate)) + "x" + + std::to_string(pad_n(p.dims.hidden)); + } // The stop tokens: `load_geometry` already copied eos/bos/pad and `architectures` out of the pack's // genai_config.json (these packs ship NO generation_config.json -- I had assumed they did), and read_dims // parsed eos into p.dims. This line is the belt: whatever the plan resolved wins over the raw metadata.