From 92d42eec6ab221fea2c11337712195bd2baa8876 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Wed, 11 Nov 2026 00:30:38 -0400 Subject: [PATCH] npu-plan: plan.json can name the wide and narrow sealed families (prefill.sealed_elf, decode.narrow_elf) (Refs #1990, #2073, #799) Plan::load reads prefill.sealed_elf / prefill.sealed_rows (17..128) and decode.narrow_elf / decode.narrow_rows (1..15), all optional, and refuses the inconsistent forms (no sealed decode set, a path instead of a bare name, rows missing or out of range, rows without a file). A loaded plan wins over config.json; NPU_WIDE_SEALED_ELF and NPU_NARROW_SEALED_ELF still override both. tools/npu_plan.py copies the config keys into the plan and writes nothing new for a pack without them. Re-applied on main after #2042 and #2074 merged; the change is the one reviewed at 4315c23 / 2c3ad10. Co-Authored-By: Claude Fable 5.2 --- docs/npu-plan.md | 8 +++ docs/npu.md | 7 ++- npu/forward/src/q4nx_forward.cpp | 9 ++++ npu/plan/plan.h | 50 +++++++++++++++++- tests/npu_plan_generate_test.py | 40 ++++++++++++++ tests/npu_plan_test.cpp | 91 ++++++++++++++++++++++++++++++++ tools/npu_plan.py | 31 ++++++++++- 7 files changed, 231 insertions(+), 5 deletions(-) diff --git a/docs/npu-plan.md b/docs/npu-plan.md index 9cef8b26a..e057310b3 100644 --- a/docs/npu-plan.md +++ b/docs/npu-plan.md @@ -56,10 +56,14 @@ tools/npu_plan.py [--kernels DIR] [--out PATH] [--check] [--print] | `container.tile_order` | q4_1 tile order; only `engine-1` is implemented. Absent means `engine-1`, so containers written before the field keep loading; an unknown id is refused. The loader is its only reader in this tree | | `fusion.gate_up` | `fused` or `split` (replaces the `full_i8_GU_*` filename probe) | | `prefill.M` | prefill activation rows; absent/0 means "not stated". Recorded and validated, but the current forward still takes its prefill rows from `Q4NX_DESIGN_M_PREFILL` (128) | +| `prefill.sealed_elf` | optional, pb16i/t2s only: a second sealed ELF, the decode set's instances built for more rows per launch (the wide prefill family, engine#1990). A bare file name in the kernels directory. Wins over the `config.json` key; `NPU_WIDE_SEALED_ELF` still overrides both. A file that is missing or does not load is logged and prefill stays at 16 rows | +| `prefill.sealed_rows` | rows per launch of `prefill.sealed_elf`, 17..128; required when it is named | | `decode.M` | decode activation rows; absent/0 means not stated. Recorded for P4’s `NpuModel`; the current forward selects the M=16 family through `decode.dirs`, not through this value | | `decode.enabled` | false disables the decode twin family | | `decode.dirs[]` | where the decode family is looked for, in order: `root` is `model` (the `-m` directory) or `kernels` (`--kernels`), `path` is relative under that root, `.` is the root itself. Empty, absolute or `..` paths are refused | | `decode.sealed_elf` | pb16i only: the sealed ELF the decode roles run | +| `decode.narrow_elf` | optional, pb16i/t2s only: a third sealed ELF, the decode set's instances built for fewer rows per launch (the narrow decode family, engine#2073). A bare file name in the kernels directory; wins over the `config.json` key, and `NPU_NARROW_SEALED_ELF` overrides both. A file that is missing or does not load is logged and decode stays on the 16-row kernels | +| `decode.narrow_rows` | rows per launch of `decode.narrow_elf`, 1..15; required when it is named | | `kernels.tag` | artifact-name suffix: `""`, `_b8`, `_pb16i`, `_t2s` or `_t2s2`; must agree with the container layout and `container.pb16i` | | `attention.mode` | `causal` (default) or `noncausal_full_attention`. The latter is a prompt-encoding mode: every full-attention layer sees the whole prompt, and the forward refuses to generate with it | | `kv.key`, `kv.value` | what the KV cache is stored as, per side: `f16` (the meaning of an absent `kv` block, and of every plan written before P7) or `int8`. `int8` is int8 with **one fp32 scale per (head, token)**, taken *after* a per-head Hadamard rotation — the rotation is what makes one scale affordable, because it collapses the per-channel magnitude spread the scale would otherwise have to cover. `key` is the side that costs no un-rotation (`(QH)(KH)ᵀ = QKᵀ`), so storing K rotated and rotating Q needs no inverse transform; `value` is a separate choice because an int8 V needs one on the output, which is why K-only is the cheap variant to prove the path with. This is a description until a target opens it: a target without the capability refuses `int8` **by name**, never serves f16 in its place. That check is `check_kv_caps` in `npu/target/target.h`, called from every target's `open()`, and it is gated device-free (`ctest -R npu_target_caps`) | @@ -103,6 +107,10 @@ The loader never substitutes a default for something it cannot honour; every ref layer count that disagrees with `model.layers`; - `container.pb16i` with an empty `decode.sealed_elf` when `kernels.tag` is `_pb16i` and decode is enabled; or a non-empty `sealed_elf` on a non-pb16i container. +- `prefill.sealed_elf` without `decode.sealed_elf`, with a path instead of a bare name, or with + `prefill.sealed_rows` missing or outside 17..128; or `prefill.sealed_rows` without `prefill.sealed_elf`. +- `decode.narrow_elf` without `decode.sealed_elf`, with a path instead of a bare name, or with + `decode.narrow_rows` missing or outside 1..15; or `decode.narrow_rows` without `decode.narrow_elf`. The forward adds one more check on load: `kernels.tag` must equal the tag the container itself implies (same block, `Q4nxNpuForward::init_impl` in `npu/forward/src/q4nx_forward.cpp`), so a plan and a container cannot diff --git a/docs/npu.md b/docs/npu.md index 85857e81f..46b69fed4 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -550,7 +550,9 @@ may name a second sealed ELF with the same instances built for more rows: `q4nx_ `q4nx_pb16i_sealed_wide_elf`) in `config.json`, a file in the kernels directory beside the 16-row one, and `q4nx_sealed_wide_rows` for its row count (default 32, allowed 17..128). No pack ships one yet, so nothing changes for an existing pack. `NPU_WIDE_SEALED_ELF=` overrides the pack's file, `NPU_WIDE_SEALED_ELF=0` -switches a pack's wide family off, and `NPU_WIDE_ROWS` overrides the row count. +switches a pack's wide family off, and `NPU_WIDE_ROWS` overrides the row count. `tools/npu_plan.py` copies +the two keys into `plan.json` as `prefill.sealed_elf` and `prefill.sealed_rows`, and a loaded plan wins over +`config.json` (`docs/npu-plan.md`). When the wide ELF loads for every role that has a 16-row kernel, the chunk width becomes its row count and a launch of more than 16 rows takes the wide kernel; decode steps and chunks of up to 16 rows stay on the @@ -583,7 +585,8 @@ takes it (a decode step, or a verify pass of a few rows); anything wider keeps t are the wide family's: all roles or none, one hw_context, shared weight images, and a file that is missing or does not load is logged (`[forward] narrow family NOT adopted (...)`) while decode stays on the 16-row kernels. `narrow_family_complete` and `narrow_takes` in `batch_width.h` are pinned by -`tests/npu_pack_test.cpp`. No pack ships a narrow ELF yet. +`tests/npu_pack_test.cpp`. No pack ships a narrow ELF yet. `plan.json` carries it as `decode.narrow_elf` and +`decode.narrow_rows` (`docs/npu-plan.md`). Measured with a 4-row set, one binary, arms 16-row / 4-row / 16-row / 4-row, three prompts generating 49 tokens each (`(tokens - 1) / (wall - wall of a 1-token request)`), two `npu-hold` tickets: diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index e3bebd1b2..ab2f24310 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -847,6 +847,15 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, // The plan is the runtime's copy of the sealed-ELF decision; when it names one it wins, so a plan // and a container cannot disagree about which file the decode roles run. if (plan_.loaded() && !plan_.sealed_elf.empty()) pb16i_sealed_ = plan_.sealed_elf; + // The plan is the pack's description when there is one, so its wide family wins over config.json's keys. + if (plan_.loaded() && !plan_.sealed_wide_elf.empty()) { + wide_sealed_ = plan_.sealed_wide_elf; + wide_sealed_rows_ = plan_.sealed_wide_rows; + } + if (plan_.loaded() && !plan_.sealed_narrow_elf.empty()) { + narrow_sealed_ = plan_.sealed_narrow_elf; + narrow_sealed_rows_ = plan_.sealed_narrow_rows; + } // Hand the layout family to the launch layer now that the plan (or the probes) has settled it. xd_->set_pb16i(pb16i_, pb16i_dir_, pb16i_sealed_); xd_->set_hadamard_block(had_block); diff --git a/npu/plan/plan.h b/npu/plan/plan.h index 643d7adf2..7fcea0a09 100644 --- a/npu/plan/plan.h +++ b/npu/plan/plan.h @@ -35,7 +35,7 @@ // "model": {"hidden": 1024, "layers": 28, "arch": "qwen3"}, // "container": {"layout": "rm"|"b8"|"pb16i"|"t2s", "chunk_kind": "mode0|mode1|mode2|pb16i", "pb16i": false}, // "fusion": {"gate_up": "fused"|"split"}, -// "prefill": {"M": 128}, +// "prefill": {"M": 128}, (+ optional "sealed_elf", "sealed_rows": a wide family) // "decode": {"M": 16, "enabled": true, "sealed_elf": "", // "dirs": [{"root": "model", "path": "kernels-m16"}, // {"root": "kernels", "path": "m16"}, @@ -206,6 +206,14 @@ struct Plan { struct DecodeDir { bool under_model = false; std::string path; }; std::vector decode_dirs; std::string sealed_elf; // pb16i: one sealed ELF, per-role instance names + // engine#1990: an optional wide prefill family -- a second sealed ELF with the same instances built for + // `sealed_wide_rows` rows per launch. Empty = none; the decode family then also runs prefill chunks. + std::string sealed_wide_elf; + int sealed_wide_rows = 0; + // engine#2073: an optional narrow decode family -- the same instances built for FEWER rows, taken by launches + // of 1..`sealed_narrow_rows` rows (a decode step carries one). Empty = none. + std::string sealed_narrow_elf; + int sealed_narrow_rows = 0; struct { bool enabled = false; int K = 0, N = 0; } moe_batched_gu; std::vector layer_plans; @@ -306,7 +314,13 @@ struct Plan { throw PlanError("plan: " + path + " fusion.gate_up \"" + gu + "\" is not fused or split"); } - if (j.contains("prefill")) p.prefill_m = positive_or_zero(obj(j, "prefill", path), "M", path); + if (j.contains("prefill")) { + const auto& pf = obj(j, "prefill", path); + p.prefill_m = positive_or_zero(pf, "M", path); + // Optional: plans written before the wide family existed load and mean what they meant. + p.sealed_wide_elf = pf.value("sealed_elf", ""); + if (pf.contains("sealed_rows")) p.sealed_wide_rows = positive_or_zero(pf, "sealed_rows", path); + } // Optional and defaulted on purpose: a plan written before this field existed must load and // mean exactly what it meant then (causal), so this is not a format bump. if (j.contains("attention")) { @@ -518,6 +532,8 @@ struct Plan { p.decode_m = positive_or_zero(d, "M", path); p.decode_enabled = d.value("enabled", true); p.sealed_elf = d.value("sealed_elf", ""); + p.sealed_narrow_elf = d.value("narrow_elf", ""); + if (d.contains("narrow_rows")) p.sealed_narrow_rows = positive_or_zero(d, "narrow_rows", path); if (d.contains("dirs")) { if (!d.at("dirs").is_array()) throw PlanError("plan: " + path + " decode.dirs must be an array"); for (const auto& one : d.at("dirs")) { @@ -587,6 +603,36 @@ struct Plan { "the decode roles run)"); if (!p.pb16i && !p.sealed_elf.empty()) throw PlanError("plan: " + path + " names decode.sealed_elf but container.pb16i is false"); + // The wide family is the decode family's instances at another width, so it cannot exist without it, and + // its row count must be one a chunk can have: more than the 16 rows of the decode family, at most 128. + if (!p.sealed_wide_elf.empty()) { + if (p.sealed_elf.empty()) + throw PlanError("plan: " + path + " names prefill.sealed_elf but decode.sealed_elf is empty (the " + "wide family is the sealed decode set built for more rows)"); + if (p.sealed_wide_elf.find('/') != std::string::npos || p.sealed_wide_elf.find("..") != std::string::npos) + throw PlanError("plan: " + path + " prefill.sealed_elf \"" + p.sealed_wide_elf + + "\" must be a bare file name in the kernels directory"); + if (p.sealed_wide_rows < 17 || p.sealed_wide_rows > 128) + throw PlanError("plan: " + path + " prefill.sealed_rows is " + std::to_string(p.sealed_wide_rows) + + " but a wide family holds 17..128 rows"); + } else if (p.sealed_wide_rows != 0) { + throw PlanError("plan: " + path + " states prefill.sealed_rows without prefill.sealed_elf"); + } + // The narrow family: the same three conditions, with the row bound on the other side of 16. + if (!p.sealed_narrow_elf.empty()) { + if (p.sealed_elf.empty()) + throw PlanError("plan: " + path + " names decode.narrow_elf but decode.sealed_elf is empty (the " + "narrow family is the sealed decode set built for fewer rows)"); + if (p.sealed_narrow_elf.find('/') != std::string::npos || + p.sealed_narrow_elf.find("..") != std::string::npos) + throw PlanError("plan: " + path + " decode.narrow_elf \"" + p.sealed_narrow_elf + + "\" must be a bare file name in the kernels directory"); + if (p.sealed_narrow_rows < 1 || p.sealed_narrow_rows > 15) + throw PlanError("plan: " + path + " decode.narrow_rows is " + std::to_string(p.sealed_narrow_rows) + + " but a narrow family holds 1..15 rows"); + } else if (p.sealed_narrow_rows != 0) { + throw PlanError("plan: " + path + " states decode.narrow_rows without decode.narrow_elf"); + } if (p.tag == "_b8" && p.layout != "b8") throw PlanError("plan: " + path + " kernels.tag is _b8 but container.layout is " + p.layout); if ((p.tag == "_t2s" || p.tag == "_t2s2") && p.layout != "t2s") diff --git a/tests/npu_plan_generate_test.py b/tests/npu_plan_generate_test.py index 14f64e41d..13722064f 100644 --- a/tests/npu_plan_generate_test.py +++ b/tests/npu_plan_generate_test.py @@ -139,6 +139,46 @@ def touch(path): check(json.load(open(os.path.join(d2c, "plan.json")))["decode"]["sealed_elf"] == "", "a t2s container does not inherit q4nx_pb16i_sealed_elf") + # 4d. engine#1990: a wide prefill family is carried into prefill{}, with its row count; a pack without + # one keeps the two-key-free block it always had; and a layout reads only its own key. + check(p2b["prefill"] == {"M": 128}, "no wide key in config -> prefill block unchanged") + d2d = model_dir(os.path.join(root, "b4"), layout="t2s", q4nx_weight_layout_version=6, + q4nx_t2s_sealed_elf="SEALED-T2S.elf", q4nx_t2s_sealed_wide_elf="SEALED-T2S-m32.elf", + layer_types=["full_attention", "full_attention"]) + run(d2d) + p2d = json.load(open(os.path.join(d2d, "plan.json"))) + check(p2d["prefill"] == {"M": 128, "sealed_elf": "SEALED-T2S-m32.elf", "sealed_rows": 32}, + "t2s wide key -> prefill.sealed_elf and the default 32 rows") + d2e = model_dir(os.path.join(root, "b5"), layout="t2s", q4nx_weight_layout_version=6, + q4nx_t2s_sealed_elf="SEALED-T2S.elf", q4nx_t2s_sealed_wide_elf="W64.elf", + q4nx_sealed_wide_rows=64, layer_types=["full_attention", "full_attention"]) + run(d2e) + check(json.load(open(os.path.join(d2e, "plan.json")))["prefill"]["sealed_rows"] == 64, + "q4nx_sealed_wide_rows is carried") + d2f = model_dir(os.path.join(root, "b6"), layout="t2s", q4nx_weight_layout_version=6, + q4nx_t2s_sealed_elf="SEALED-T2S.elf", q4nx_pb16i_sealed_wide_elf="PB-WIDE.elf", + layer_types=["full_attention", "full_attention"]) + run(d2f) + check(json.load(open(os.path.join(d2f, "plan.json")))["prefill"] == {"M": 128}, + "a t2s container does not inherit q4nx_pb16i_sealed_wide_elf") + + # 4e. engine#2073: the narrow decode family is carried into decode{} the same way. + check("narrow_elf" not in p2b["decode"] and "narrow_rows" not in p2b["decode"], + "no narrow key in config -> decode block unchanged") + d2g = model_dir(os.path.join(root, "b7"), layout="t2s", q4nx_weight_layout_version=6, + q4nx_t2s_sealed_elf="SEALED-T2S.elf", q4nx_t2s_sealed_narrow_elf="SEALED-T2S-m4.elf", + layer_types=["full_attention", "full_attention"]) + run(d2g) + dg = json.load(open(os.path.join(d2g, "plan.json")))["decode"] + check(dg.get("narrow_elf") == "SEALED-T2S-m4.elf" and dg.get("narrow_rows") == 4, + "t2s narrow key -> decode.narrow_elf and the default 4 rows") + d2h = model_dir(os.path.join(root, "b8"), layout="t2s", q4nx_weight_layout_version=6, + q4nx_t2s_sealed_elf="SEALED-T2S.elf", q4nx_pb16i_sealed_narrow_elf="PB-NARROW.elf", + layer_types=["full_attention", "full_attention"]) + run(d2h) + check("narrow_elf" not in json.load(open(os.path.join(d2h, "plan.json")))["decode"], + "a t2s container does not inherit q4nx_pb16i_sealed_narrow_elf") + # 5. The plan printer writes nothing. r = run(d2, "--print") check(r.returncode == 0 and '"format": "1bit-npu-plan/1"' in r.stdout, "--print emits the plan on stdout") diff --git a/tests/npu_plan_test.cpp b/tests/npu_plan_test.cpp index c9518f318..c1d0992c1 100644 --- a/tests/npu_plan_test.cpp +++ b/tests/npu_plan_test.cpp @@ -91,6 +91,95 @@ bool has(const std::string& hay, const std::string& needle) { return hay.find(needle) != std::string::npos; } +// engine#1990: prefill.sealed_elf / prefill.sealed_rows, the wide family. +void test_wide_family() { + auto pb = base_plan(); + pb["container"]["layout"] = "pb16i"; + pb["container"]["chunk_kind"] = "pb16i"; + pb["container"]["pb16i"] = true; + pb["kernels"]["tag"] = "_pb16i"; + pb["decode"]["sealed_elf"] = "S16.elf"; + const std::string base_refusal = refusal(write_plan("wide_base", pb)); + check(base_refusal.empty(), "the pb16i base plan of the wide arms loads (" + base_refusal + ")"); + const Plan none = Plan::load(write_plan("wide_none", pb)); + check(none.sealed_wide_elf.empty() && none.sealed_wide_rows == 0, "no wide family unless the plan names one"); + + auto w = pb; + w["prefill"]["sealed_elf"] = "S32.elf"; + w["prefill"]["sealed_rows"] = 32; + const Plan wp = Plan::load(write_plan("wide_ok", w)); + check(wp.sealed_wide_elf == "S32.elf" && wp.sealed_wide_rows == 32 && wp.prefill_m == 128, + "prefill.sealed_elf and sealed_rows are read, prefill.M is untouched"); + + auto norows = pb; + norows["prefill"]["sealed_elf"] = "S32.elf"; + check(has(refusal(write_plan("wide_norows", norows)), "17..128"), "a wide family without a row count is refused"); + for (int bad : {16, 129}) { + auto b = w; + b["prefill"]["sealed_rows"] = bad; + check(has(refusal(write_plan("wide_rows", b)), "17..128"), + "sealed_rows " + std::to_string(bad) + " is refused"); + } + auto b17 = w; + b17["prefill"]["sealed_rows"] = 17; + check(refusal(write_plan("wide_17", b17)).empty(), "sealed_rows 17 loads"); + auto orphan = pb; + orphan["prefill"]["sealed_rows"] = 32; + check(has(refusal(write_plan("wide_orphan", orphan)), "without prefill.sealed_elf"), + "sealed_rows without sealed_elf is refused"); + auto nodecode = base_plan(); + nodecode["prefill"]["sealed_elf"] = "S32.elf"; + nodecode["prefill"]["sealed_rows"] = 32; + check(has(refusal(write_plan("wide_nodecode", nodecode)), "decode.sealed_elf is empty"), + "a wide family without the sealed decode set is refused"); + for (const char* name : {"sub/S32.elf", "../S32.elf", "/abs/S32.elf"}) { + auto b = w; + b["prefill"]["sealed_elf"] = name; + check(has(refusal(write_plan("wide_path", b)), "bare file name"), std::string("path ") + name + " is refused"); + } +} + +// engine#2073: decode.narrow_elf / decode.narrow_rows, the narrow family. +void test_narrow_family() { + auto pb = base_plan(); + pb["container"]["layout"] = "pb16i"; + pb["container"]["chunk_kind"] = "pb16i"; + pb["container"]["pb16i"] = true; + pb["kernels"]["tag"] = "_pb16i"; + pb["decode"]["sealed_elf"] = "S16.elf"; + const Plan none = Plan::load(write_plan("narrow_none", pb)); + check(none.sealed_narrow_elf.empty() && none.sealed_narrow_rows == 0, "no narrow family unless the plan names one"); + auto n = pb; + n["decode"]["narrow_elf"] = "S4.elf"; + n["decode"]["narrow_rows"] = 4; + const Plan np = Plan::load(write_plan("narrow_ok", n)); + check(np.sealed_narrow_elf == "S4.elf" && np.sealed_narrow_rows == 4 && np.sealed_elf == "S16.elf", + "decode.narrow_elf and narrow_rows are read beside the sealed set"); + auto norows = pb; + norows["decode"]["narrow_elf"] = "S4.elf"; + check(has(refusal(write_plan("narrow_norows", norows)), "1..15"), "a narrow family without a row count is refused"); + auto b16 = n; + b16["decode"]["narrow_rows"] = 16; + check(has(refusal(write_plan("narrow_16", b16)), "1..15"), "narrow_rows 16 is refused"); + for (int ok : {1, 15}) { + auto b = n; + b["decode"]["narrow_rows"] = ok; + check(refusal(write_plan("narrow_edge", b)).empty(), "narrow_rows " + std::to_string(ok) + " loads"); + } + auto orphan = pb; + orphan["decode"]["narrow_rows"] = 4; + check(has(refusal(write_plan("narrow_orphan", orphan)), "without decode.narrow_elf"), + "narrow_rows without narrow_elf is refused"); + auto nosealed = base_plan(); + nosealed["decode"]["narrow_elf"] = "S4.elf"; + nosealed["decode"]["narrow_rows"] = 4; + check(has(refusal(write_plan("narrow_nosealed", nosealed)), "decode.sealed_elf is empty"), + "a narrow family without the sealed decode set is refused"); + auto path = n; + path["decode"]["narrow_elf"] = "sub/S4.elf"; + check(has(refusal(write_plan("narrow_path", path)), "bare file name"), "a path as narrow_elf is refused"); +} + void test_valid() { const std::string path = write_plan("valid", base_plan()); const Plan p = Plan::load(path); @@ -695,6 +784,8 @@ void test_drafter_and_state() { int main() { test_valid(); + test_wide_family(); + test_narrow_family(); test_rope_and_inputs(); test_drafter_and_state(); test_kv(); diff --git a/tools/npu_plan.py b/tools/npu_plan.py index 0ac82c278..f49eb5ceb 100644 --- a/tools/npu_plan.py +++ b/tools/npu_plan.py @@ -210,6 +210,34 @@ def decode_dirs(model_dir: str, kernels: str) -> list[dict]: return dirs + +def narrow_fields(cfg, layout): + """decode.narrow_elf / decode.narrow_rows when the container's config names a narrow family (engine#2073). + + Written only when the pack has one, so every other pack's plan is byte-for-byte what it was. + """ + if layout not in ("pb16i", "t2s"): + return {} + narrow = cfg.get("q4nx_t2s_sealed_narrow_elf" if layout == "t2s" else "q4nx_pb16i_sealed_narrow_elf", "") + if not narrow: + return {} + return {"narrow_elf": narrow, "narrow_rows": int(cfg.get("q4nx_sealed_narrow_rows", 4))} + + +def prefill_block(cfg, layout): + """prefill.M, plus the wide sealed family when the container's config names one (engine#1990). + + The two extra fields are written only when the pack has a wide family, so the plan of every pack + without one is byte-for-byte what it was. Like decode.sealed_elf, each layout reads its own key. + """ + block = {"M": 128} + if layout in ("pb16i", "t2s"): + wide = cfg.get("q4nx_t2s_sealed_wide_elf" if layout == "t2s" else "q4nx_pb16i_sealed_wide_elf", "") + if wide: + block["sealed_elf"] = wide + block["sealed_rows"] = int(cfg.get("q4nx_sealed_wide_rows", 32)) + return block + def tile_order_claim(model_dir: str) -> tuple[str, str]: """(the tile_order this plan may declare, a note to print) from the directory's repack stamp. @@ -313,7 +341,7 @@ def generate(model_dir: str, kernels: str) -> dict: "pb16i": layout in ("pb16i", "t2s"), }, "fusion": {"gate_up": "fused" if fused_gu else "split"}, - "prefill": {"M": 128}, + "prefill": prefill_block(cfg, layout), "decode": { "M": 16, "enabled": True, @@ -324,6 +352,7 @@ def generate(model_dir: str, kernels: str) -> dict: # +2.0-2.3 ms, more than the kernels themselves. "sealed_elf": cfg.get("q4nx_t2s_sealed_elf" if layout == "t2s" else "q4nx_pb16i_sealed_elf", "") if layout in ("pb16i", "t2s") else "", + **narrow_fields(cfg, layout), }, "kernels": {"tag": tag}, "attention": {"mode": attention_mode(model_dir)},