diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 8a5015c2d..fcff4a8ed 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -3482,9 +3482,121 @@ static std::string group_suffix_slot(const std::string& role, const std::string& return std::string(); } +// engine#1788, 3c: WHY THIS IS A BYTE COPY AND MUST STAY ONE. npu/q4nx.h records that the Q8 chunk (8704 B) comes in TWO +// code layouts and that nothing in the chunk or the container says which: the forward's decoder picks per tensor from the +// codes (q4nx_q8_split16 -- in the right layout every group's absmax code is 127). So a writer that re-encoded or +// re-quantised would have to make a choice the container does not make, and getting it wrong is a pack that decodes to +// wrong weights with no error anywhere. Copying the payload's bytes verbatim cannot make that mistake: the layout is the +// container's, and the forward already knows how to read it. +WritePackResult container_write_pack(const std::string& payload_path, const ImportPlan& plan, + const std::string& out_dir, bool allow_unplaced) { + WritePackResult r; + r.payload = out_dir + "/model.q4nx"; + r.header = out_dir + "/pack.json"; + if (!plan.unplaced.empty() && !allow_unplaced) { + r.why = std::to_string(plan.unplaced.size()) + " tensor(s) have no destination (e.g. " + plan.unplaced.front() + + "); refusing to write a partial pack"; + return r; + } + std::ifstream in(payload_path, std::ios::binary); + if (!in) { r.why = "cannot open the payload: " + payload_path; return r; } + std::error_code ec; + std::filesystem::create_directories(out_dir, ec); + if (ec) { r.why = "cannot create " + out_dir + ": " + ec.message(); return r; } + std::ofstream out(r.payload, std::ios::binary | std::ios::trunc); + if (!out) { r.why = "cannot write " + r.payload; return r; } + json tensors = json::object(); + int64_t off = 0; + for (const auto& item : plan.items) { + in.seekg(plan.payload_base + item.begin); + std::string bytes(size_t(item.end - item.begin), '\0'); + if (!in.read(&bytes[0], std::streamsize(bytes.size()))) { + r.why = "short read at " + std::to_string(plan.payload_base + item.begin) + " for " + item.source; return r; + } + out.write(bytes.data(), std::streamsize(bytes.size())); + if (!out) { r.why = "short write to " + r.payload; return r; } + // the header records where each slot's bytes sit in the contiguous payload: the offsets ARE the format + tensors[item.slot] = {{"offset", off}, {"bytes", int64_t(bytes.size())}, + {"shape", item.shape}, {"dtype", item.dtype}, {"source", item.source}}; + off += int64_t(bytes.size()); + ++r.tensors; + } + out.close(); + if (!out) { r.why = "short write to " + r.payload; return r; } + json h; + h["tensors"] = tensors; + h["bytes"] = off; + h["payload_bytes"] = plan.payload_bytes; + h["unplaced"] = plan.unplaced; + std::ofstream hout(r.header, std::ios::trunc); + if (!hout) { r.why = "cannot write " + r.header; return r; } + hout << h.dump(2) << "\n"; + if (!hout) { r.why = "short write to " + r.header; return r; } + r.bytes = off; + r.ok = true; + return r; +} + +WriteLayoutResult container_write_layout(const std::string& payload_path, const ImportPlan& plan, + const std::string& out_dir, bool allow_unplaced) { + WriteLayoutResult r; + if (!plan.unplaced.empty() && !allow_unplaced) { + r.why = std::to_string(plan.unplaced.size()) + " tensor(s) have no destination (e.g. " + plan.unplaced.front() + + "); refusing to write a partial pack"; + return r; + } + std::ifstream in(payload_path, std::ios::binary); + if (!in) { r.why = "cannot open the payload: " + payload_path; return r; } + json manifest = json::object(); + std::error_code ec; + std::filesystem::create_directories(out_dir, ec); + if (ec) { r.why = "cannot create " + out_dir + ": " + ec.message(); return r; } + for (const auto& item : plan.items) { + in.seekg(plan.payload_base + item.begin); + std::string bytes(size_t(item.end - item.begin), '\0'); + if (!in.read(&bytes[0], std::streamsize(bytes.size()))) { + r.why = "short read at " + std::to_string(plan.payload_base + item.begin) + " for " + item.source; + return r; + } + const std::string path = out_dir + "/" + item.slot; + std::ofstream out(path, std::ios::binary | std::ios::trunc); + if (!out) { r.why = "cannot write " + path; return r; } + out.write(bytes.data(), std::streamsize(bytes.size())); + if (!out) { r.why = "short write to " + path; return r; } + ++r.files; + r.bytes += int64_t(bytes.size()); + manifest[item.slot] = {{"source", item.source}, + {"bytes", int64_t(bytes.size())}, + {"shape", item.shape}, + {"dtype", item.dtype}}; + } + // The layout is self-describing: a directory of files whose names are engine slots is not enough, because the reader + // also needs to know which container tensor each came from and what shape/dtype it is. + { + const std::string mpath = out_dir + "/manifest.json"; + std::ofstream mout(mpath, std::ios::trunc); + if (!mout) { r.why = "cannot write " + mpath; return r; } + json j; + j["tensors"] = manifest; + j["files"] = r.files; + j["bytes"] = r.bytes; + j["payload_bytes"] = plan.payload_bytes; + j["unplaced"] = plan.unplaced; + mout << j.dump(2) << "\n"; + if (!mout) { r.why = "short write to " + mpath; return r; } + } + r.ok = true; + return r; +} + ImportPlan container_import_plan(const Q4nxContainer& container, const std::vector& layer_types) { ImportPlan out; out.payload_bytes = container.payload_bytes; + // `header_bytes` is the JSON length alone; the file prefixes it with an 8-byte little-endian length, so the payload + // starts at 8 + header_bytes. Getting this by 8 wrong is what made the first end-to-end run write header bytes. + // ... and there is no prefix when the container declares no header at all (a bare payload has header_bytes 0), so the + // +8 only applies to a real safetensors-shaped container. + out.payload_base = container.header_bytes > 0 ? container.header_bytes + 8 : 0; const PlacementReport rep = container_placement(container.tensors, layer_types); std::map by_name; for (const auto& t : container.tensors) by_name[t.name] = &t; @@ -5081,10 +5193,32 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons // the first byte fails with "cannot open /model.onnx" -- a missing-file message about a pack that has no graph // at all. Refused by name, the same way the external-data genre below is, until the container-genre writer exists. if (p.container_tensors >= 0) { - throw std::runtime_error( - "this pack is a safetensors container (model.q4nx): " + std::to_string(p.container_tensors) + - " tensor(s); importing it needs the container-genre writer (engine#1786, item F1), which the importer does " - "not have yet. `1bit npu-pack info` reports what the container holds"); + // engine#1788: the writer exists now. The payload is copied, never re-quantised -- the container's bytes ARE the + // q4nx chunks (a tensor shaped [4096,8,4736] I8 is 4096 rows of the 4736-byte Q4_K chunk), and copying is the only + // safe answer to q4nx.h's two-Q8-layouts problem: the bytes never change, so the choice the container does not make + // never has to be made. + const std::string multi = pack_dir + "/model.q4nx"; + const Q4nxContainer c = read_q4nx_container(multi); + if (!c.why.empty()) throw std::runtime_error("container: " + c.why); + // the layer types live in config.json beside the container, not in its safetensors metadata + std::string cfg_source; + const json cfg = load_geometry(pack_dir, &cfg_source); + std::vector lt; + const json* src = cfg.contains("layer_types") ? &cfg : nullptr; + if (!src && cfg.contains("text_config") && cfg["text_config"].is_object() && cfg["text_config"].contains("layer_types")) + src = &cfg["text_config"]; + if (src && (*src)["layer_types"].is_array()) + for (const auto& x : (*src)["layer_types"]) if (x.is_string()) lt.push_back(x.get()); + const ImportPlan ip = container_import_plan(c, lt); + const WritePackResult w = container_write_pack(multi, ip, out_dir, opt.allow_incomplete); + if (!w.ok) throw std::runtime_error("container: " + w.why); + Report rep; + rep.out_dir = out_dir; + rep.tensors = int(w.tensors); + rep.tiles = w.bytes; + // the unplaced list is a REFUSAL list, so it goes where the caller already looks for refusals + for (const auto& u : ip.unplaced) rep.refused.push_back(u); + return rep; } if (p.graph_packed_siblings > 0) { // The genre has a writer now (engine#1503): decode -> convert -> split by role -> the same I8 tile chunks, diff --git a/npu/pack_import.h b/npu/pack_import.h index cc3b0f238..f577b6f35 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -428,9 +428,45 @@ struct ImportPlan { std::vector unplaced; // ": " -- a refusal, not a silent skip int64_t bytes = 0; // payload bytes the items carry int64_t payload_bytes = 0; // what the container says it has, so a writer can prove coverage + // engine#1788, 3c: where the payload STARTS in the container file. `Q4nxTensor::begin/end` are PAYLOAD-relative, so a + // writer that seeks to `begin` in the file reads the safetensors header instead of the tensor -- which is exactly what + // happened: the bytes came out right in total (1024+128) and wrong in content. + int64_t payload_base = 0; }; ImportPlan container_import_plan(const Q4nxContainer& container, const std::vector& layer_types); +// engine#1788, 3c: the I/O half -- copy each planned byte range out of the container's payload into one file per +// destination slot, plus a manifest. It REFUSES to write when the container has tensors the plan could not place, unless +// `allow_unplaced` is set: a pack that silently omits a fifth of the model is worse than no pack, and the caller is the +// only one who can decide that an omission is intended. +struct WriteLayoutResult { + bool ok = false; + int64_t files = 0; + int64_t bytes = 0; + std::string why; // why it refused, or what failed, when ok is false +}; +WriteLayoutResult container_write_layout(const std::string& payload_path, const ImportPlan& plan, + const std::string& out_dir, bool allow_unplaced); +// The PACK form: one contiguous payload plus a header, which is the shape the forward opens (a container of the same genre). +// Same refusal as the layout form -- nothing is written when the plan has unplaced tensors and allow_unplaced is false -- +// because a header that omits a fifth of the model is exactly as silent as a missing file. +struct WritePackResult { + bool ok = false; + int64_t tensors = 0; + int64_t bytes = 0; + std::string payload; // the file written, when ok + std::string header; + std::string why; +}; +WritePackResult container_write_pack(const std::string& payload_path, const ImportPlan& plan, + const std::string& out_dir, bool allow_unplaced); +// engine#1788, 3c: what the import should EMIT, learned from the source rather than invented. A container tensor's shape is +// already the chunk: `[4096,8,4736] I8` is N=4096, 8 groups, 4736 bytes per row, and 4736 is exactly the Q4_K chunk size. +// So the payload IS q4nx chunk data and the import needs no re-quantisation -- it emits a container OF THE SAME GENRE: +// one contiguous payload (the planned byte ranges in order) plus a header naming each destination slot with its dtype, +// shape and offset. That is also the only safe answer to q4nx.h's two-Q8-layouts problem: the bytes never change. +// `container_write_layout` above is the debugging form (one file per slot); the pack form concatenates instead. + // engine#1788, slice 3c: the writer's per-TENSOR decision, which is what an import actually iterates. Reads the tensor's // own layer, takes that layer's declared type, and asks the table above for a destination. A tensor with no role and a // tensor with a role but no destination are counted apart, because the first is a vocabulary gap and the second is a diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 997d6b047..7e393254c 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -5432,6 +5432,124 @@ void test_container_import_plan() { check(okp.unplaced.empty() && okp.bytes == 24, "import: disjoint ranges out of order are fine"); } +// engine#1788, 3c: the I/O half. Writes real files to a scratch dir and reads them back -- the only way to know the byte +// ranges are the right bytes. +void test_container_write_layout() { + namespace fs = std::filesystem; + using pi::Q4nxTensor; + const std::string dir = (fs::temp_directory_path() / "onebit-write-layout-test").string(); + fs::remove_all(dir); fs::create_directories(dir); + const std::string payload = dir + "/payload.bin"; + { std::ofstream o(payload, std::ios::binary); for (int i = 0; i < 64; ++i) o.put(char('A' + (i % 26))); } + + pi::Q4nxContainer c; + const auto t = [](const std::string& n, int64_t b, int64_t e) { + Q4nxTensor x; x.name = n; x.shape = {2, 2}; x.dtype = "I8"; x.begin = b; x.end = e; return x; + }; + c.tensors = {t("model.embed_tokens.weight", 0, 8), t("model.layers.0.self_attn.q_proj.weight", 8, 24)}; + c.payload_bytes = 64; + const pi::ImportPlan plan = pi::container_import_plan(c, {"full_attention"}); + + const std::string out = dir + "/out"; + const pi::WriteLayoutResult w = pi::container_write_layout(payload, plan, out, false); + check(w.ok, "write: a complete plan writes"); + check(w.files == 2, "write: one file per placed tensor"); + check(w.bytes == 24, "write: and the bytes are the ranges, not the payload"); + { + std::ifstream i1(out + "/model.embed_tokens.weight", std::ios::binary); + std::string s1((std::istreambuf_iterator(i1)), std::istreambuf_iterator()); + check(s1 == "ABCDEFGH", "write: the embedding carries ITS bytes (0..8)"); + std::ifstream i2(out + "/model.layers.0.self_attn.q_proj.weight", std::ios::binary); + std::string s2((std::istreambuf_iterator(i2)), std::istreambuf_iterator()); + check(s2 == "IJKLMNOPQRSTUVWX", "write: the projection carries ITS bytes (8..24)"); + std::ifstream mi(out + "/manifest.json"); + const std::string mtext((std::istreambuf_iterator(mi)), std::istreambuf_iterator()); + check(mtext.find("model.layers.0.self_attn.q_proj.weight") != std::string::npos, + "write: the manifest names the slot"); + check(mtext.find("\"source\": \"model.layers.0.self_attn.q_proj.weight\"") != std::string::npos, + "write: and the container tensor it came from"); + } + + // the refusal: an unplaced tensor means the pack would silently omit part of the model + pi::Q4nxContainer bad; + bad.tensors = {t("model.embed_tokens.weight", 0, 8), t("model.layers.0.mystery.weight", 8, 16)}; + bad.payload_bytes = 64; + const pi::ImportPlan bp = pi::container_import_plan(bad, {}); + const std::string out2 = dir + "/out2"; + const pi::WriteLayoutResult w2 = pi::container_write_layout(payload, bp, out2, false); + check(!w2.ok, "write: an incomplete plan is refused"); + check(w2.why.find("no destination") != std::string::npos, "write: and says why"); + check(!fs::exists(out2 + "/model.embed_tokens.weight"), "write: and writes NOTHING, not a partial pack"); + // ... unless the caller says the omission is intended + const pi::WriteLayoutResult w3 = pi::container_write_layout(payload, bp, out2, true); + check(w3.ok && w3.files == 1, "write: allow_unplaced writes the placed part and no more"); + fs::remove_all(dir); +} + +// engine#1788, 3c: the PACK form. One contiguous payload plus a header of offsets -- so the arm reads the payload back at +// each header offset and compares with the source range. An offset header that is merely self-consistent is not enough. +void test_container_write_pack() { + namespace fs = std::filesystem; + using pi::Q4nxTensor; + const std::string dir = (fs::temp_directory_path() / "onebit-write-pack-test").string(); + fs::remove_all(dir); fs::create_directories(dir); + const std::string payload = dir + "/payload.bin"; + { std::ofstream o(payload, std::ios::binary); for (int i = 0; i < 64; ++i) o.put(char('A' + (i % 26))); } + + pi::Q4nxContainer c; + const auto t = [](const std::string& n, int64_t b, int64_t e) { + Q4nxTensor x; x.name = n; x.shape = {2, 2}; x.dtype = "I8"; x.begin = b; x.end = e; return x; + }; + c.tensors = {t("model.embed_tokens.weight", 0, 8), t("model.layers.0.self_attn.q_proj.weight", 8, 24)}; + c.payload_bytes = 64; + const pi::ImportPlan plan = pi::container_import_plan(c, {"full_attention"}); + + const std::string out = dir + "/pack"; + // REVIEW of the first end-to-end run: the ranges are PAYLOAD-relative, so a container with a non-zero header must be + // read from payload_base. This arm only means anything if the payload has a header in front of it, which is why the + // fixture below prepends one -- the earlier arms could not have caught the bug because their base was 0. + const std::string with_hdr = dir + "/payload_with_header.bin"; + { std::ofstream o(with_hdr, std::ios::binary); for (int i = 0; i < 24; ++i) o.put(char('0' + (i % 10))); } + { std::ofstream o(with_hdr, std::ios::binary | std::ios::app); for (int i = 0; i < 64; ++i) o.put(char('A' + (i % 26))); } + pi::Q4nxContainer hc; + hc.tensors = {t("model.embed_tokens.weight", 0, 8), t("model.layers.0.self_attn.q_proj.weight", 8, 24)}; + hc.payload_bytes = 64; + hc.header_bytes = 16; // the JSON length; the payload starts at 8 + this, after the length prefix + const pi::ImportPlan hplan = pi::container_import_plan(hc, {"full_attention"}); + check(hplan.payload_base == 24, "pack: the plan carries where the payload starts (8 + the JSON length)"); + const pi::WritePackResult hw = pi::container_write_pack(with_hdr, hplan, dir + "/pack_hdr", false); + check(hw.ok, "pack: a container with a header writes"); + { + std::ifstream bi(dir + "/pack_hdr/model.q4nx", std::ios::binary); + const std::string blob((std::istreambuf_iterator(bi)), std::istreambuf_iterator()); + check(blob == "ABCDEFGHIJKLMNOPQRSTUVWX", + "pack: and the bytes are the PAYLOAD's, not the header's (got: " + blob.substr(0, 12) + ")"); + } + + const pi::WritePackResult w = pi::container_write_pack(payload, plan, out, false); + check(w.ok, "pack: writes"); + check(w.tensors == 2 && w.bytes == 24, "pack: two tensors, 24 bytes"); + { + std::ifstream pi_in(out + "/model.q4nx", std::ios::binary); + const std::string blob((std::istreambuf_iterator(pi_in)), std::istreambuf_iterator()); + check(blob.size() == 24, "pack: the payload is exactly the planned bytes"); + check(blob.substr(0, 8) == "ABCDEFGH", "pack: the first slot sits at offset 0"); + check(blob.substr(8, 16) == "IJKLMNOPQRSTUVWX", "pack: the second follows contiguously"); + std::ifstream hi(out + "/pack.json"); + const std::string htext((std::istreambuf_iterator(hi)), std::istreambuf_iterator()); + check(htext.find("\"offset\": 0") != std::string::npos, "pack: the header states offset 0"); + check(htext.find("\"offset\": 8") != std::string::npos, "pack: and offset 8 for the second"); + } + // same refusal as the layout form + pi::Q4nxContainer bad; + bad.tensors = {t("model.embed_tokens.weight", 0, 8), t("model.layers.0.mystery.weight", 8, 16)}; + bad.payload_bytes = 64; + const pi::ImportPlan bp = pi::container_import_plan(bad, {}); + const pi::WritePackResult w2 = pi::container_write_pack(payload, bp, dir + "/pack2", false); + check(!w2.ok && w2.why.find("no destination") != std::string::npos, "pack: an incomplete plan is refused with a reason"); + fs::remove_all(dir); +} + void test_container_placement() { using pi::Q4nxTensor; const auto t = [](const std::string& n) { Q4nxTensor x; x.name = n; x.dtype = "I8"; return x; }; @@ -6715,6 +6833,8 @@ int main(int argc, char** argv) { test_container_roles_gemma_local(); test_container_roles_moe_and_singular_layer(); test_container_placement(); + test_container_write_pack(); + test_container_write_layout(); test_container_import_plan(); test_scale_and_proj_norm_roles(); test_role_tables_disagree_nowhere();