From 981975a2bb45af72e004e0f7d1e7952b884d0f51 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 12 Nov 2026 19:56:33 -0400 Subject: [PATCH 1/7] npu-pack: the I/O half -- write the layout, and refuse to write a partial pack (engine#1788, 3c) Copies each planned byte range out of the container's payload into one file per destination slot. The refusal is the point: when the plan has tensors with no destination it writes NOTHING unless the caller passes allow_unplaced, because a pack that silently omits a fifth of the model is worse than no pack and only the caller can decide an omission is intended. Armed by writing to a scratch directory and reading the files back, which is the only way to know the ranges are the right bytes: embedding 0..8 -> "ABCDEFGH" q_proj 8..24 -> "IJKLMNOPQRSTUVWX" and the refusal arm asserts three things at once -- not ok, a reason that names the problem, and no file on disk. Mutant (drop the unplaced guard): FAIL write: an incomplete plan is refused FAIL write: and says why FAIL write: and writes NOTHING, not a partial pack (cherry picked from commit 75bb5622cbbf61c27f75411796f3dd9103b355da) --- npu/pack_import.cpp | 32 ++++++++++++++++++++++ npu/pack_import.h | 13 +++++++++ tests/npu_pack_import_test.cpp | 49 ++++++++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 8a5015c2d..57fd65172 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -3482,6 +3482,38 @@ static std::string group_suffix_slot(const std::string& role, const std::string& return std::string(); } +WriteLayoutResult container_write_layout(const std::string& payload_path, const ImportPlan& plan, + const std::string& out_dir, bool allow_unplaced) { + WriteLayoutResult r; + if (!plan.unplaced.empty() && !allow_unplaced) { + r.why = std::to_string(plan.unplaced.size()) + " tensor(s) have no destination (e.g. " + plan.unplaced.front() + + "); refusing to write a partial pack"; + return r; + } + std::ifstream in(payload_path, std::ios::binary); + if (!in) { r.why = "cannot open the payload: " + payload_path; return r; } + std::error_code ec; + std::filesystem::create_directories(out_dir, ec); + if (ec) { r.why = "cannot create " + out_dir + ": " + ec.message(); return r; } + for (const auto& item : plan.items) { + in.seekg(item.begin); + std::string bytes(size_t(item.end - item.begin), '\0'); + if (!in.read(&bytes[0], std::streamsize(bytes.size()))) { + r.why = "short read at " + std::to_string(item.begin) + " for " + item.source; + return r; + } + const std::string path = out_dir + "/" + item.slot; + std::ofstream out(path, std::ios::binary | std::ios::trunc); + if (!out) { r.why = "cannot write " + path; return r; } + out.write(bytes.data(), std::streamsize(bytes.size())); + if (!out) { r.why = "short write to " + path; return r; } + ++r.files; + r.bytes += int64_t(bytes.size()); + } + r.ok = true; + return r; +} + ImportPlan container_import_plan(const Q4nxContainer& container, const std::vector& layer_types) { ImportPlan out; out.payload_bytes = container.payload_bytes; diff --git a/npu/pack_import.h b/npu/pack_import.h index cc3b0f238..07daecfbe 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -431,6 +431,19 @@ struct ImportPlan { }; ImportPlan container_import_plan(const Q4nxContainer& container, const std::vector& layer_types); +// engine#1788, 3c: the I/O half -- copy each planned byte range out of the container's payload into one file per +// destination slot, plus a manifest. It REFUSES to write when the container has tensors the plan could not place, unless +// `allow_unplaced` is set: a pack that silently omits a fifth of the model is worse than no pack, and the caller is the +// only one who can decide that an omission is intended. +struct WriteLayoutResult { + bool ok = false; + int64_t files = 0; + int64_t bytes = 0; + std::string why; // why it refused, or what failed, when ok is false +}; +WriteLayoutResult container_write_layout(const std::string& payload_path, const ImportPlan& plan, + const std::string& out_dir, bool allow_unplaced); + // engine#1788, slice 3c: the writer's per-TENSOR decision, which is what an import actually iterates. Reads the tensor's // own layer, takes that layer's declared type, and asks the table above for a destination. A tensor with no role and a // tensor with a role but no destination are counted apart, because the first is a vocabulary gap and the second is a diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 997d6b047..1679eca46 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -5432,6 +5432,54 @@ void test_container_import_plan() { check(okp.unplaced.empty() && okp.bytes == 24, "import: disjoint ranges out of order are fine"); } +// engine#1788, 3c: the I/O half. Writes real files to a scratch dir and reads them back -- the only way to know the byte +// ranges are the right bytes. +void test_container_write_layout() { + namespace fs = std::filesystem; + using pi::Q4nxTensor; + const std::string dir = (fs::temp_directory_path() / "onebit-write-layout-test").string(); + fs::remove_all(dir); fs::create_directories(dir); + const std::string payload = dir + "/payload.bin"; + { std::ofstream o(payload, std::ios::binary); for (int i = 0; i < 64; ++i) o.put(char('A' + (i % 26))); } + + pi::Q4nxContainer c; + const auto t = [](const std::string& n, int64_t b, int64_t e) { + Q4nxTensor x; x.name = n; x.shape = {2, 2}; x.dtype = "I8"; x.begin = b; x.end = e; return x; + }; + c.tensors = {t("model.embed_tokens.weight", 0, 8), t("model.layers.0.self_attn.q_proj.weight", 8, 24)}; + c.payload_bytes = 64; + const pi::ImportPlan plan = pi::container_import_plan(c, {"full_attention"}); + + const std::string out = dir + "/out"; + const pi::WriteLayoutResult w = pi::container_write_layout(payload, plan, out, false); + check(w.ok, "write: a complete plan writes"); + check(w.files == 2, "write: one file per placed tensor"); + check(w.bytes == 24, "write: and the bytes are the ranges, not the payload"); + { + std::ifstream i1(out + "/model.embed_tokens.weight", std::ios::binary); + std::string s1((std::istreambuf_iterator(i1)), std::istreambuf_iterator()); + check(s1 == "ABCDEFGH", "write: the embedding carries ITS bytes (0..8)"); + std::ifstream i2(out + "/model.layers.0.self_attn.q_proj.weight", std::ios::binary); + std::string s2((std::istreambuf_iterator(i2)), std::istreambuf_iterator()); + check(s2 == "IJKLMNOPQRSTUVWX", "write: the projection carries ITS bytes (8..24)"); + } + + // the refusal: an unplaced tensor means the pack would silently omit part of the model + pi::Q4nxContainer bad; + bad.tensors = {t("model.embed_tokens.weight", 0, 8), t("model.layers.0.mystery.weight", 8, 16)}; + bad.payload_bytes = 64; + const pi::ImportPlan bp = pi::container_import_plan(bad, {}); + const std::string out2 = dir + "/out2"; + const pi::WriteLayoutResult w2 = pi::container_write_layout(payload, bp, out2, false); + check(!w2.ok, "write: an incomplete plan is refused"); + check(w2.why.find("no destination") != std::string::npos, "write: and says why"); + check(!fs::exists(out2 + "/model.embed_tokens.weight"), "write: and writes NOTHING, not a partial pack"); + // ... unless the caller says the omission is intended + const pi::WriteLayoutResult w3 = pi::container_write_layout(payload, bp, out2, true); + check(w3.ok && w3.files == 1, "write: allow_unplaced writes the placed part and no more"); + fs::remove_all(dir); +} + void test_container_placement() { using pi::Q4nxTensor; const auto t = [](const std::string& n) { Q4nxTensor x; x.name = n; x.dtype = "I8"; return x; }; @@ -6715,6 +6763,7 @@ int main(int argc, char** argv) { test_container_roles_gemma_local(); test_container_roles_moe_and_singular_layer(); test_container_placement(); + test_container_write_layout(); test_container_import_plan(); test_scale_and_proj_norm_roles(); test_role_tables_disagree_nowhere(); From 75737e8aa3ab459aeb2eb50a9a0cd4a4021abe64 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 12 Nov 2026 20:01:56 -0400 Subject: [PATCH 2/7] npu-pack: the layout writes a manifest, so it is self-describing (engine#1788, 3c) A directory of files whose names are engine slots is not enough: a reader also needs to know which container tensor each came from and what shape and dtype it is. So the write emits manifest.json alongside the files, carrying per slot the source tensor, byte count, shape and dtype, plus the plan totals (files, bytes, payload_bytes) and the unplaced list. That last field matters: a layout written with allow_unplaced says so IN THE OUTPUT, so a later reader cannot mistake a deliberately partial layout for a complete one. (cherry picked from commit 972b841691a020eb54ea1fc385c48f9f3426e6e0) --- npu/pack_import.cpp | 20 ++++++++++++++++++++ tests/npu_pack_import_test.cpp | 6 ++++++ 2 files changed, 26 insertions(+) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 57fd65172..13cbe00c9 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -3492,6 +3492,7 @@ WriteLayoutResult container_write_layout(const std::string& payload_path, const } std::ifstream in(payload_path, std::ios::binary); if (!in) { r.why = "cannot open the payload: " + payload_path; return r; } + json manifest = json::object(); std::error_code ec; std::filesystem::create_directories(out_dir, ec); if (ec) { r.why = "cannot create " + out_dir + ": " + ec.message(); return r; } @@ -3509,6 +3510,25 @@ WriteLayoutResult container_write_layout(const std::string& payload_path, const if (!out) { r.why = "short write to " + path; return r; } ++r.files; r.bytes += int64_t(bytes.size()); + manifest[item.slot] = {{"source", item.source}, + {"bytes", int64_t(bytes.size())}, + {"shape", item.shape}, + {"dtype", item.dtype}}; + } + // The layout is self-describing: a directory of files whose names are engine slots is not enough, because the reader + // also needs to know which container tensor each came from and what shape/dtype it is. + { + const std::string mpath = out_dir + "/manifest.json"; + std::ofstream mout(mpath, std::ios::trunc); + if (!mout) { r.why = "cannot write " + mpath; return r; } + json j; + j["tensors"] = manifest; + j["files"] = r.files; + j["bytes"] = r.bytes; + j["payload_bytes"] = plan.payload_bytes; + j["unplaced"] = plan.unplaced; + mout << j.dump(2) << "\n"; + if (!mout) { r.why = "short write to " + mpath; return r; } } r.ok = true; return r; diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 1679eca46..f48a74673 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -5462,6 +5462,12 @@ void test_container_write_layout() { std::ifstream i2(out + "/model.layers.0.self_attn.q_proj.weight", std::ios::binary); std::string s2((std::istreambuf_iterator(i2)), std::istreambuf_iterator()); check(s2 == "IJKLMNOPQRSTUVWX", "write: the projection carries ITS bytes (8..24)"); + std::ifstream mi(out + "/manifest.json"); + const std::string mtext((std::istreambuf_iterator(mi)), std::istreambuf_iterator()); + check(mtext.find("model.layers.0.self_attn.q_proj.weight") != std::string::npos, + "write: the manifest names the slot"); + check(mtext.find("\"source\": \"model.layers.0.self_attn.q_proj.weight\"") != std::string::npos, + "write: and the container tensor it came from"); } // the refusal: an unplaced tensor means the pack would silently omit part of the model From 625e206f3e5d9f1170fcb53a8364fa09f4bff811 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 12 Nov 2026 20:10:50 -0400 Subject: [PATCH 3/7] npu-pack: record why the layout is a byte copy and must stay one (engine#1788, 3c) npu/q4nx.h documents that the Q8 chunk comes in TWO code layouts -- byte k*32+r, or (r/16)*4096 + g*512 + (k%32)*16 + r%16 -- and that NOTHING in the chunk or the container says which. The forward's decoder picks per tensor from the codes (q4nx_q8_split16: in the right layout every group's absmax code is 127), and the two real models on this box use different ones. That makes the central design constraint of the writer explicit at the place it would be violated: a writer that re-encoded or re-quantised would have to make a choice the container does not make, and getting it wrong produces a pack that decodes to wrong weights with no error anywhere. Copying the payload verbatim cannot make that mistake. (cherry picked from commit 603a278ec3ca6ecb8649dd23aa889069d1f646fa) --- npu/pack_import.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 13cbe00c9..a3b4c2f0d 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -3482,6 +3482,12 @@ static std::string group_suffix_slot(const std::string& role, const std::string& return std::string(); } +// engine#1788, 3c: WHY THIS IS A BYTE COPY AND MUST STAY ONE. npu/q4nx.h records that the Q8 chunk (8704 B) comes in TWO +// code layouts and that nothing in the chunk or the container says which: the forward's decoder picks per tensor from the +// codes (q4nx_q8_split16 -- in the right layout every group's absmax code is 127). So a writer that re-encoded or +// re-quantised would have to make a choice the container does not make, and getting it wrong is a pack that decodes to +// wrong weights with no error anywhere. Copying the payload's bytes verbatim cannot make that mistake: the layout is the +// container's, and the forward already knows how to read it. WriteLayoutResult container_write_layout(const std::string& payload_path, const ImportPlan& plan, const std::string& out_dir, bool allow_unplaced) { WriteLayoutResult r; From e7bfcae76f80ee467ccdfbfc0ee9e7973d6f756b Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 12 Nov 2026 20:20:47 -0400 Subject: [PATCH 4/7] npu-pack: the container import emits a container of the same genre (engine#1788, 3c) Recorded at the declaration, because it decides the next code step. A container tensor's shape is ALREADY the chunk -- [4096,8,4736] I8 is N=4096, 8 groups, 4736 bytes per row, and 4736 is exactly the Q4_K chunk size -- so the payload is q4nx chunk data and the import needs no re-quantisation. It emits a container of the same genre: one contiguous payload (the planned ranges in order) plus a header naming each destination slot with dtype, shape and offset. That is also the only safe answer to q4nx.h's two-Q8-layouts problem: the bytes never change, so the choice the container does not make never has to be made. container_write_layout() stays as the debugging form (one file per slot); the pack form concatenates instead. (cherry picked from commit 9e022123cfe776379a34c5639c4f85f563f0179d) --- npu/pack_import.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/npu/pack_import.h b/npu/pack_import.h index 07daecfbe..168a14dc0 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -443,6 +443,12 @@ struct WriteLayoutResult { }; WriteLayoutResult container_write_layout(const std::string& payload_path, const ImportPlan& plan, const std::string& out_dir, bool allow_unplaced); +// engine#1788, 3c: what the import should EMIT, learned from the source rather than invented. A container tensor's shape is +// already the chunk: `[4096,8,4736] I8` is N=4096, 8 groups, 4736 bytes per row, and 4736 is exactly the Q4_K chunk size. +// So the payload IS q4nx chunk data and the import needs no re-quantisation -- it emits a container OF THE SAME GENRE: +// one contiguous payload (the planned byte ranges in order) plus a header naming each destination slot with its dtype, +// shape and offset. That is also the only safe answer to q4nx.h's two-Q8-layouts problem: the bytes never change. +// `container_write_layout` above is the debugging form (one file per slot); the pack form concatenates instead. // engine#1788, slice 3c: the writer's per-TENSOR decision, which is what an import actually iterates. Reads the tensor's // own layer, takes that layer's declared type, and asks the table above for a destination. A tensor with no role and a From 5562aeffa3cd6c1326b42543e6341a67e2851f9c Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 12 Nov 2026 20:43:21 -0400 Subject: [PATCH 5/7] npu-pack: the pack form -- one contiguous payload and a header of offsets (engine#1788, 3c) The shape the forward opens, and the last piece of the writer. Writes model.q4nx as the planned byte ranges concatenated in order, and pack.json as the header: per slot its offset, byte count, shape, dtype and source tensor, plus the totals and the unplaced list. Called the container genre because the payload IS the container's chunks, copied -- nothing is re-quantised (see the note above container_write_layout on why that is the only safe choice). Armed by reading the payload back AT THE HEADER'S OFFSETS and comparing against the source ranges, because a header that is merely self-consistent proves nothing: offset 0 -> "ABCDEFGH" (model.embed_tokens.weight, source 0..8) offset 8 -> "IJKLMNOPQRSTUVWX" (model.layers.0.self_attn.q_proj.weight, source 8..24) Mutant (leave a one-byte gap between slots): "pack: two tensors, 24 bytes" and "pack: and offset 8 for the second" both fail. And the same refusal as the layout form: an incomplete plan writes nothing and says how many tensors had no destination. (cherry picked from commit 4876df3a52cede519eac6d544cf0614c19bfc8b5) --- npu/pack_import.cpp | 49 ++++++++++++++++++++++++++++++++++ npu/pack_import.h | 13 +++++++++ tests/npu_pack_import_test.cpp | 44 ++++++++++++++++++++++++++++++ 3 files changed, 106 insertions(+) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index a3b4c2f0d..807dfff7b 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -3488,6 +3488,55 @@ static std::string group_suffix_slot(const std::string& role, const std::string& // re-quantised would have to make a choice the container does not make, and getting it wrong is a pack that decodes to // wrong weights with no error anywhere. Copying the payload's bytes verbatim cannot make that mistake: the layout is the // container's, and the forward already knows how to read it. +WritePackResult container_write_pack(const std::string& payload_path, const ImportPlan& plan, + const std::string& out_dir, bool allow_unplaced) { + WritePackResult r; + r.payload = out_dir + "/model.q4nx"; + r.header = out_dir + "/pack.json"; + if (!plan.unplaced.empty() && !allow_unplaced) { + r.why = std::to_string(plan.unplaced.size()) + " tensor(s) have no destination (e.g. " + plan.unplaced.front() + + "); refusing to write a partial pack"; + return r; + } + std::ifstream in(payload_path, std::ios::binary); + if (!in) { r.why = "cannot open the payload: " + payload_path; return r; } + std::error_code ec; + std::filesystem::create_directories(out_dir, ec); + if (ec) { r.why = "cannot create " + out_dir + ": " + ec.message(); return r; } + std::ofstream out(r.payload, std::ios::binary | std::ios::trunc); + if (!out) { r.why = "cannot write " + r.payload; return r; } + json tensors = json::object(); + int64_t off = 0; + for (const auto& item : plan.items) { + in.seekg(item.begin); + std::string bytes(size_t(item.end - item.begin), '\0'); + if (!in.read(&bytes[0], std::streamsize(bytes.size()))) { + r.why = "short read at " + std::to_string(item.begin) + " for " + item.source; return r; + } + out.write(bytes.data(), std::streamsize(bytes.size())); + if (!out) { r.why = "short write to " + r.payload; return r; } + // the header records where each slot's bytes sit in the contiguous payload: the offsets ARE the format + tensors[item.slot] = {{"offset", off}, {"bytes", int64_t(bytes.size())}, + {"shape", item.shape}, {"dtype", item.dtype}, {"source", item.source}}; + off += int64_t(bytes.size()); + ++r.tensors; + } + out.close(); + if (!out) { r.why = "short write to " + r.payload; return r; } + json h; + h["tensors"] = tensors; + h["bytes"] = off; + h["payload_bytes"] = plan.payload_bytes; + h["unplaced"] = plan.unplaced; + std::ofstream hout(r.header, std::ios::trunc); + if (!hout) { r.why = "cannot write " + r.header; return r; } + hout << h.dump(2) << "\n"; + if (!hout) { r.why = "short write to " + r.header; return r; } + r.bytes = off; + r.ok = true; + return r; +} + WriteLayoutResult container_write_layout(const std::string& payload_path, const ImportPlan& plan, const std::string& out_dir, bool allow_unplaced) { WriteLayoutResult r; diff --git a/npu/pack_import.h b/npu/pack_import.h index 168a14dc0..bffe0632d 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -443,6 +443,19 @@ struct WriteLayoutResult { }; WriteLayoutResult container_write_layout(const std::string& payload_path, const ImportPlan& plan, const std::string& out_dir, bool allow_unplaced); +// The PACK form: one contiguous payload plus a header, which is the shape the forward opens (a container of the same genre). +// Same refusal as the layout form -- nothing is written when the plan has unplaced tensors and allow_unplaced is false -- +// because a header that omits a fifth of the model is exactly as silent as a missing file. +struct WritePackResult { + bool ok = false; + int64_t tensors = 0; + int64_t bytes = 0; + std::string payload; // the file written, when ok + std::string header; + std::string why; +}; +WritePackResult container_write_pack(const std::string& payload_path, const ImportPlan& plan, + const std::string& out_dir, bool allow_unplaced); // engine#1788, 3c: what the import should EMIT, learned from the source rather than invented. A container tensor's shape is // already the chunk: `[4096,8,4736] I8` is N=4096, 8 groups, 4736 bytes per row, and 4736 is exactly the Q4_K chunk size. // So the payload IS q4nx chunk data and the import needs no re-quantisation -- it emits a container OF THE SAME GENRE: diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index f48a74673..e3e433e68 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -5486,6 +5486,49 @@ void test_container_write_layout() { fs::remove_all(dir); } +// engine#1788, 3c: the PACK form. One contiguous payload plus a header of offsets -- so the arm reads the payload back at +// each header offset and compares with the source range. An offset header that is merely self-consistent is not enough. +void test_container_write_pack() { + namespace fs = std::filesystem; + using pi::Q4nxTensor; + const std::string dir = (fs::temp_directory_path() / "onebit-write-pack-test").string(); + fs::remove_all(dir); fs::create_directories(dir); + const std::string payload = dir + "/payload.bin"; + { std::ofstream o(payload, std::ios::binary); for (int i = 0; i < 64; ++i) o.put(char('A' + (i % 26))); } + + pi::Q4nxContainer c; + const auto t = [](const std::string& n, int64_t b, int64_t e) { + Q4nxTensor x; x.name = n; x.shape = {2, 2}; x.dtype = "I8"; x.begin = b; x.end = e; return x; + }; + c.tensors = {t("model.embed_tokens.weight", 0, 8), t("model.layers.0.self_attn.q_proj.weight", 8, 24)}; + c.payload_bytes = 64; + const pi::ImportPlan plan = pi::container_import_plan(c, {"full_attention"}); + + const std::string out = dir + "/pack"; + const pi::WritePackResult w = pi::container_write_pack(payload, plan, out, false); + check(w.ok, "pack: writes"); + check(w.tensors == 2 && w.bytes == 24, "pack: two tensors, 24 bytes"); + { + std::ifstream pi_in(out + "/model.q4nx", std::ios::binary); + const std::string blob((std::istreambuf_iterator(pi_in)), std::istreambuf_iterator()); + check(blob.size() == 24, "pack: the payload is exactly the planned bytes"); + check(blob.substr(0, 8) == "ABCDEFGH", "pack: the first slot sits at offset 0"); + check(blob.substr(8, 16) == "IJKLMNOPQRSTUVWX", "pack: the second follows contiguously"); + std::ifstream hi(out + "/pack.json"); + const std::string htext((std::istreambuf_iterator(hi)), std::istreambuf_iterator()); + check(htext.find("\"offset\": 0") != std::string::npos, "pack: the header states offset 0"); + check(htext.find("\"offset\": 8") != std::string::npos, "pack: and offset 8 for the second"); + } + // same refusal as the layout form + pi::Q4nxContainer bad; + bad.tensors = {t("model.embed_tokens.weight", 0, 8), t("model.layers.0.mystery.weight", 8, 16)}; + bad.payload_bytes = 64; + const pi::ImportPlan bp = pi::container_import_plan(bad, {}); + const pi::WritePackResult w2 = pi::container_write_pack(payload, bp, dir + "/pack2", false); + check(!w2.ok && w2.why.find("no destination") != std::string::npos, "pack: an incomplete plan is refused with a reason"); + fs::remove_all(dir); +} + void test_container_placement() { using pi::Q4nxTensor; const auto t = [](const std::string& n) { Q4nxTensor x; x.name = n; x.dtype = "I8"; return x; }; @@ -6769,6 +6812,7 @@ int main(int argc, char** argv) { test_container_roles_gemma_local(); test_container_roles_moe_and_singular_layer(); test_container_placement(); + test_container_write_pack(); test_container_write_layout(); test_container_import_plan(); test_scale_and_proj_norm_roles(); From 77e1b15a180f93da70e2899583bb0d394f0140e8 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 12 Nov 2026 21:23:09 -0400 Subject: [PATCH 6/7] npu-pack: the container import is wired -- 1bit npu-pack import now writes a container (engine#1788, 3c) The refusal that has been there since #2277 ("importing it needs the container-genre writer, which the importer does not have yet") is replaced by the writer: read the container, take the layer types from config.json beside it, plan the placement, and write model.q4nx + pack.json. The payload is COPIED, never re-quantised, which is the only safe answer to q4nx.h's two-Q8-layouts problem. End to end on the fixture pack (840 KB, no device, no 21 GB write at the current disk floor): $ 1bit npu-pack import .../container-genre -o out $ ls out -> model.q4nx pack.json pack.json: bytes, payload_bytes, unplaced, and per slot offset/bytes/shape/dtype/source The unplaced list goes into Report::refused, because it IS a refusal list and the caller already prints that. With allow_incomplete the write proceeds and the header records what was left out; without it, nothing is written at all. And a trap worth recording: the first run of this wrote only model.q4nx and I nearly concluded the branch had not fired -- ninja had no work to do for `1bit` because I had built `onebit`, and the binary was stale. Fifth time tonight; the fix is always to check the binary's timestamp before believing its output. (cherry picked from commit ac53ceffab3dbd3c333b1208b4365b670b210ffd) --- npu/pack_import.cpp | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 807dfff7b..f4ca6c5ce 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -5188,10 +5188,32 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons // the first byte fails with "cannot open /model.onnx" -- a missing-file message about a pack that has no graph // at all. Refused by name, the same way the external-data genre below is, until the container-genre writer exists. if (p.container_tensors >= 0) { - throw std::runtime_error( - "this pack is a safetensors container (model.q4nx): " + std::to_string(p.container_tensors) + - " tensor(s); importing it needs the container-genre writer (engine#1786, item F1), which the importer does " - "not have yet. `1bit npu-pack info` reports what the container holds"); + // engine#1788: the writer exists now. The payload is copied, never re-quantised -- the container's bytes ARE the + // q4nx chunks (a tensor shaped [4096,8,4736] I8 is 4096 rows of the 4736-byte Q4_K chunk), and copying is the only + // safe answer to q4nx.h's two-Q8-layouts problem: the bytes never change, so the choice the container does not make + // never has to be made. + const std::string multi = pack_dir + "/model.q4nx"; + const Q4nxContainer c = read_q4nx_container(multi); + if (!c.why.empty()) throw std::runtime_error("container: " + c.why); + // the layer types live in config.json beside the container, not in its safetensors metadata + std::string cfg_source; + const json cfg = load_geometry(pack_dir, &cfg_source); + std::vector lt; + const json* src = cfg.contains("layer_types") ? &cfg : nullptr; + if (!src && cfg.contains("text_config") && cfg["text_config"].is_object() && cfg["text_config"].contains("layer_types")) + src = &cfg["text_config"]; + if (src && (*src)["layer_types"].is_array()) + for (const auto& x : (*src)["layer_types"]) if (x.is_string()) lt.push_back(x.get()); + const ImportPlan ip = container_import_plan(c, lt); + const WritePackResult w = container_write_pack(multi, ip, out_dir, opt.allow_incomplete); + if (!w.ok) throw std::runtime_error("container: " + w.why); + Report rep; + rep.out_dir = out_dir; + rep.tensors = int(w.tensors); + rep.tiles = w.bytes; + // the unplaced list is a REFUSAL list, so it goes where the caller already looks for refusals + for (const auto& u : ip.unplaced) rep.refused.push_back(u); + return rep; } if (p.graph_packed_siblings > 0) { // The genre has a writer now (engine#1503): decode -> convert -> split by role -> the same I8 tile chunks, From 950b327844823022f224b56ef050443f2e4abf6d Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 12 Nov 2026 22:32:22 -0400 Subject: [PATCH 7/7] npu-pack: seek from the PAYLOAD base, not the file start (engine#1788, 3c) The first end-to-end run wrote the right TOTAL (1024+128 = 1152 bytes) and the wrong BYTES: Q4nxTensor::begin/end are payload-relative, so seeking to `begin` in the file read the safetensors header. `header_bytes` is the JSON length alone, so the payload starts at 8 + header_bytes -- and that +8 only exists when there IS a header (a bare payload has header_bytes 0 and no prefix), which is why the fix is conditional. Verified by comparing every written byte against the source container's payload: $ 1bit npu-pack import .../container-genre -o out OK model.embed_tokens.weight <- model.embed_tokens.weight OK model.layers.0.mlp.down_proj.weight <- model.layers.0.mlp.down_proj.weight EVERY BYTE MATCHES: True out size 1152 == pack.json bytes 1152 and armed so it cannot regress: a fixture whose payload has a 24-byte header must produce the PAYLOAD's bytes, and the mutant (seek without the base) fails it with exactly the symptom production had -- "got: 012345678901", the header bytes. The arm that could have caught this the first time is the one with a NON-ZERO header: the earlier arms all had base 0, so a missing base was invisible to every one of them. (cherry picked from commit 0517978b344d3934d6353ae6ab7c9dd978888ccf) --- npu/pack_import.cpp | 13 +++++++++---- npu/pack_import.h | 4 ++++ tests/npu_pack_import_test.cpp | 21 +++++++++++++++++++++ 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index f4ca6c5ce..fcff4a8ed 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -3508,10 +3508,10 @@ WritePackResult container_write_pack(const std::string& payload_path, const Impo json tensors = json::object(); int64_t off = 0; for (const auto& item : plan.items) { - in.seekg(item.begin); + in.seekg(plan.payload_base + item.begin); std::string bytes(size_t(item.end - item.begin), '\0'); if (!in.read(&bytes[0], std::streamsize(bytes.size()))) { - r.why = "short read at " + std::to_string(item.begin) + " for " + item.source; return r; + r.why = "short read at " + std::to_string(plan.payload_base + item.begin) + " for " + item.source; return r; } out.write(bytes.data(), std::streamsize(bytes.size())); if (!out) { r.why = "short write to " + r.payload; return r; } @@ -3552,10 +3552,10 @@ WriteLayoutResult container_write_layout(const std::string& payload_path, const std::filesystem::create_directories(out_dir, ec); if (ec) { r.why = "cannot create " + out_dir + ": " + ec.message(); return r; } for (const auto& item : plan.items) { - in.seekg(item.begin); + in.seekg(plan.payload_base + item.begin); std::string bytes(size_t(item.end - item.begin), '\0'); if (!in.read(&bytes[0], std::streamsize(bytes.size()))) { - r.why = "short read at " + std::to_string(item.begin) + " for " + item.source; + r.why = "short read at " + std::to_string(plan.payload_base + item.begin) + " for " + item.source; return r; } const std::string path = out_dir + "/" + item.slot; @@ -3592,6 +3592,11 @@ WriteLayoutResult container_write_layout(const std::string& payload_path, const ImportPlan container_import_plan(const Q4nxContainer& container, const std::vector& layer_types) { ImportPlan out; out.payload_bytes = container.payload_bytes; + // `header_bytes` is the JSON length alone; the file prefixes it with an 8-byte little-endian length, so the payload + // starts at 8 + header_bytes. Getting this by 8 wrong is what made the first end-to-end run write header bytes. + // ... and there is no prefix when the container declares no header at all (a bare payload has header_bytes 0), so the + // +8 only applies to a real safetensors-shaped container. + out.payload_base = container.header_bytes > 0 ? container.header_bytes + 8 : 0; const PlacementReport rep = container_placement(container.tensors, layer_types); std::map by_name; for (const auto& t : container.tensors) by_name[t.name] = &t; diff --git a/npu/pack_import.h b/npu/pack_import.h index bffe0632d..f577b6f35 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -428,6 +428,10 @@ struct ImportPlan { std::vector unplaced; // ": " -- a refusal, not a silent skip int64_t bytes = 0; // payload bytes the items carry int64_t payload_bytes = 0; // what the container says it has, so a writer can prove coverage + // engine#1788, 3c: where the payload STARTS in the container file. `Q4nxTensor::begin/end` are PAYLOAD-relative, so a + // writer that seeks to `begin` in the file reads the safetensors header instead of the tensor -- which is exactly what + // happened: the bytes came out right in total (1024+128) and wrong in content. + int64_t payload_base = 0; }; ImportPlan container_import_plan(const Q4nxContainer& container, const std::vector& layer_types); diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index e3e433e68..7e393254c 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -5505,6 +5505,27 @@ void test_container_write_pack() { const pi::ImportPlan plan = pi::container_import_plan(c, {"full_attention"}); const std::string out = dir + "/pack"; + // REVIEW of the first end-to-end run: the ranges are PAYLOAD-relative, so a container with a non-zero header must be + // read from payload_base. This arm only means anything if the payload has a header in front of it, which is why the + // fixture below prepends one -- the earlier arms could not have caught the bug because their base was 0. + const std::string with_hdr = dir + "/payload_with_header.bin"; + { std::ofstream o(with_hdr, std::ios::binary); for (int i = 0; i < 24; ++i) o.put(char('0' + (i % 10))); } + { std::ofstream o(with_hdr, std::ios::binary | std::ios::app); for (int i = 0; i < 64; ++i) o.put(char('A' + (i % 26))); } + pi::Q4nxContainer hc; + hc.tensors = {t("model.embed_tokens.weight", 0, 8), t("model.layers.0.self_attn.q_proj.weight", 8, 24)}; + hc.payload_bytes = 64; + hc.header_bytes = 16; // the JSON length; the payload starts at 8 + this, after the length prefix + const pi::ImportPlan hplan = pi::container_import_plan(hc, {"full_attention"}); + check(hplan.payload_base == 24, "pack: the plan carries where the payload starts (8 + the JSON length)"); + const pi::WritePackResult hw = pi::container_write_pack(with_hdr, hplan, dir + "/pack_hdr", false); + check(hw.ok, "pack: a container with a header writes"); + { + std::ifstream bi(dir + "/pack_hdr/model.q4nx", std::ios::binary); + const std::string blob((std::istreambuf_iterator(bi)), std::istreambuf_iterator()); + check(blob == "ABCDEFGHIJKLMNOPQRSTUVWX", + "pack: and the bytes are the PAYLOAD's, not the header's (got: " + blob.substr(0, 12) + ")"); + } + const pi::WritePackResult w = pi::container_write_pack(payload, plan, out, false); check(w.ok, "pack: writes"); check(w.tensors == 2 && w.bytes == 24, "pack: two tensors, 24 bytes");