diff --git a/npu/forward/include/model.h b/npu/forward/include/model.h index 86b60ab0a..05ddb7346 100644 --- a/npu/forward/include/model.h +++ b/npu/forward/include/model.h @@ -18,6 +18,7 @@ #include #include +#include #include "common.h" #ifdef __cplusplus @@ -130,6 +131,33 @@ void* model_tensor_data(ModelWeights* mw, TensorDesc* desc); // Used for architecture-specific tensors the fixed fields do not cover. // (Distinct from the legacy model_find_tensor(name, mw), which returns an index.) TensorDesc* model_tensor_by_name(ModelWeights* mw, const char* name); +// engine#1786 (item F1): three FastFlowLM NPU2 model sources spell the per-layer prefix `model.layer.` (singular) +// where this forward addresses `model.layers.`. Writes the OTHER spelling into `out` and returns 1 when `name` carries +// either prefix, 0 when it does not. Only the leading prefix is swapped, once: a caller that wants a fallback retries +// the lookup with this name AFTER an exact match has failed, so a container carrying both spellings still resolves +// exactly first. Pure string work, exported so a test can pin it without a container. +static inline int model_tensor_name_alt(const char* name, char* out, size_t cap) { + static const char kPlural[] = "model.layers."; + static const char kSingular[] = "model.layer."; + const size_t nl = sizeof(kPlural) - 1, ns = sizeof(kSingular) - 1; + const char* to; + size_t fl; + if (!name || !out || cap == 0) return 0; + if (strncmp(name, kPlural, nl) == 0) { /* model.layers. -> model.layer. */ + to = kSingular; fl = nl; /* fl is the MATCHED prefix's length, not the target's */ + } else if (strncmp(name, kSingular, ns) == 0) { /* model.layer. -> model.layers. */ + to = kPlural; fl = ns; + } else { + return 0; + } + const char* rest = name + fl; /* everything after the prefix, unchanged */ + const size_t rl = strlen(rest); + const size_t tl = (to == kPlural) ? nl : ns; + if (tl + rl + 1 > cap) return 0; + memcpy(out, to, tl); + memcpy(out + tl, rest, rl + 1); + return 1; +} void model_print_info(ModelWeights* mw); int model_find_tensor(const char* name, ModelWeights* mw); diff --git a/npu/forward/src/model.c b/npu/forward/src/model.c index a5983c9c3..21a3d119c 100644 --- a/npu/forward/src/model.c +++ b/npu/forward/src/model.c @@ -1157,12 +1157,24 @@ void model_free(ModelWeights* mw) { free(mw); } +// engine#1786 (item F1): the per-layer prefix, in the two spellings the NPU2 model sources use. `layers.` is this +// forward's own spelling; `layer.` is what three of the fifty-eight containers carry (`model.layer.0.input_layernorm…`), +// where every other tensor matches by name. Swapping one prefix is a spelling fallback, not a rename: it happens only +// after an exact match has failed, so a container that carries both is unaffected. + TensorDesc* model_tensor_by_name(ModelWeights* mw, const char* name) { if (!mw || !mw->all_tensors || !name) return NULL; for (int i = 0; i < mw->all_count; i++) { const char* n = mw->all_tensors[i].name; if (n[0] && strcmp(n, name) == 0) return &mw->all_tensors[i]; } + // Exact match failed: try the other per-layer spelling before reporting absence (engine#1786). + char alt[512]; + if (!model_tensor_name_alt(name, alt, sizeof(alt))) return NULL; + for (int i = 0; i < mw->all_count; i++) { + const char* n = mw->all_tensors[i].name; + if (n[0] && strcmp(n, alt) == 0) return &mw->all_tensors[i]; + } return NULL; } diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 2fb204870..78bc8fa7e 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -1704,6 +1704,85 @@ bool fconst_set_block_headers(FconstGeom& g, const std::vector& inner) std::string find_token_fconst_exported(const std::string& pack_dir) { return find_token_fconst(pack_dir); } +// engine#1786 (item F1): read a FastFlowLM NPU2 model source's container header. Safetensors layout: an 8-byte +// little-endian header length, then that many bytes of JSON. `__metadata__` is a string map, not a tensor, and the +// vendor's converter does not always write one -- so its presence is not a genre test and its absence is not an error. +// Every failure names what was wrong, because the caller's next move ("refuse this pack" vs "read it") depends on which. +Q4nxContainer read_q4nx_container(const std::string& path) { + Q4nxContainer out; + std::ifstream f(path, std::ios::binary); + if (!f) { + out.why = "cannot open " + path; + return out; + } + unsigned char len[8] = {0}; + if (!f.read(reinterpret_cast(len), 8)) { + out.why = path + ": shorter than the 8-byte safetensors header length"; + return out; + } + int64_t n = 0; + for (int i = 7; i >= 0; --i) n = (n << 8) | int64_t(len[i]); + // A header this large is not a container: the largest real one seen on the NPU2 sources is 26,496 B (237 tensors). + // Refusing above 64 MiB keeps a mis-typed length from making the caller allocate whatever the file claims. + if (n <= 0 || n > (int64_t(64) << 20)) { + out.why = path + ": implausible header length " + std::to_string(n) + " B"; + return out; + } + std::string hdr(size_t(n), '\0'); + if (!f.read(&hdr[0], n)) { + out.why = path + ": truncated header (" + std::to_string(n) + " B claimed)"; + return out; + } + out.header_bytes = n; + json j; + try { + j = json::parse(hdr); + } catch (const std::exception& e) { + out.why = path + ": header is not JSON (" + e.what() + ")"; + return out; + } + if (!j.is_object()) { + out.why = path + ": header is not a JSON object"; + return out; + } + for (auto it = j.begin(); it != j.end(); ++it) { + if (it.key() == "__metadata__") continue; + if (!it.value().is_object()) { + out.why = path + ": tensor '" + it.key() + "' is not an object"; + out.tensors.clear(); + return out; + } + Q4nxTensor t; + t.name = it.key(); + const json& v = it.value(); + t.dtype = v.value("dtype", std::string()); + if (t.dtype.empty()) { + out.why = path + ": tensor '" + t.name + "' states no dtype"; + out.tensors.clear(); + return out; + } + if (v.contains("shape") && v["shape"].is_array()) + for (const json& d : v["shape"]) t.shape.push_back(d.is_number_integer() ? d.get() : 0); + if (!v.contains("data_offsets") || !v["data_offsets"].is_array() || v["data_offsets"].size() != 2 || + !v["data_offsets"][0].is_number_integer() || !v["data_offsets"][1].is_number_integer()) { + out.why = path + ": tensor '" + t.name + "' has no [begin, end] data_offsets"; + out.tensors.clear(); + return out; + } + t.begin = v["data_offsets"][0].get(); + t.end = v["data_offsets"][1].get(); + if (t.begin < 0 || t.end < t.begin) { + out.why = path + ": tensor '" + t.name + "' has data_offsets [" + std::to_string(t.begin) + ", " + + std::to_string(t.end) + "]"; + out.tensors.clear(); + return out; + } + out.payload_bytes += t.end - t.begin; + out.tensors.push_back(std::move(t)); + } + return out; +} + // engine#1638: the geometry READ (not just its dims), so a test can assert where each number came from -- the // text_config lift and the top-level-wins rule are both properties of load_geometry(), not of read_dims(). json load_geometry_exported(const std::string& pack_dir, std::string* source) { return load_geometry(pack_dir, source); } @@ -2840,6 +2919,18 @@ std::string Plan::manifest() const { {"rule", "length == N * (K/128 + 1) * 68"}}; j["graph_weights"] = g; } + // engine#1786 (item F1): a FastFlowLM NPU2 model source has no graph, so the census above is absent and this is + // what `info` reports instead -- a count and a dtype map taken from the container's own header, plus what the genre + // still needs. Measured on FastFlowLM/Gemma3-270M-NPU2 (0.54 GB): 237 tensors, 0.503 GB, {BF16: 110, I8: 127}. + if (container_tensors >= 0) { + json c; + c["format"] = "safetensors (model.q4nx)"; + c["tensors"] = container_tensors; + c["payload_bytes"] = container_payload_bytes; + c["dtypes"] = container_dtypes; + c["note"] = container_note; + j["container"] = c; + } j["mlp_section"] = {{"gate_up_units_measured", mlp_gu_units}, {"down_groups_measured", mlp_down_groups}, {"note", mlp_note}}; @@ -2891,6 +2982,29 @@ Plan plan(const std::string& pack_dir) { p.dir = pack_dir; p.pack = fs::path(pack_dir).lexically_normal().filename().string(); + // engine#1786 (item F1): the FastFlowLM NPU2 model sources are a safetensors container named model.q4nx with no + // graph at all. Detect that genre before anything looks for a graph, report what the container holds, and say what + // it still needs -- the alternative is the first byte failing with "cannot open /model.onnx", which describes + // a missing file on a pack that has no graph by design. + { + std::error_code ec; + const std::string container = pack_dir + "/model.q4nx"; + const std::string graph_now = graph_filename(pack_dir); + if (fs::exists(container, ec) && !fs::exists(pack_dir + "/" + graph_now, ec)) { + const Q4nxContainer c = read_q4nx_container(container); + p.container_tensors = int64_t(c.tensors.size()); + p.container_payload_bytes = c.payload_bytes; + for (const auto& t : c.tensors) p.container_dtypes[t.dtype]++; + p.container_note = c.why.empty() + ? "the pack is a safetensors container (model.q4nx): " + std::to_string(c.tensors.size()) + + " tensors, " + std::to_string(c.payload_bytes) + " B of payload, no graph. Reading it as a" + " weights source needs the container-genre plan (engine#1786, item F1); until then this pack is" + " recognised and reported, not importable." + : "model.q4nx is present but unreadable: " + c.why; + return p; + } + } + std::string geom_src; json geom = load_geometry(pack_dir, &geom_src); // `intermediate_size` is the one key no 1.6/1.7 pack states: genai_config.json has no such field, and on the diff --git a/npu/pack_import.h b/npu/pack_import.h index 7ec92ac36..1f8263de7 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -208,6 +208,14 @@ struct Plan { // third-party parser: the rai_1.3/1.6/1.7.1 line ships its 4-bit weights as ordinary ONNX initializers, while // 1.8 and R1-Distill ship placeholders and keep the weights in model.bin (pi-inferzo, 2026-11-08). // graph_matmuls < 0 means "no graph was read", which is why it is not just a count. + // engine#1786 (item F1): a FastFlowLM NPU2 model source is a safetensors container (`model.q4nx`) with no graph, + // so the two census fields above stay -1. `container_tensors` is -1 for every pack that is not that genre; when it + // is not, it is the container's tensor count and the dtype map says what the reader found. Reported rather than + // refused at the first byte, because "cannot open model.onnx" says nothing about a pack that has no graph by design. + int64_t container_tensors = -1; + int64_t container_payload_bytes = 0; + std::map container_dtypes; + std::string container_note; // what the container genre still needs, empty for a non-container pack int64_t graph_matmuls = -1; // MatMulNBits nodes in the main graph int64_t graph_real_weights = 0; // of those, how many point at an initializer with dims (i.e. real weights) int64_t graph_external = 0; // initializers that declare external data (location + offset) @@ -251,6 +259,29 @@ int intermediate_size_from_graph(const std::string& model_path, int64_t hidden); // multi-component one (`type: compositemodel`), whose geometry files stay at the root. Exported for the unit test. std::string find_token_fconst_exported(const std::string& pack_dir); +// A FastFlowLM NPU2 model source (engine#1786, item F1) is a safetensors container named `model.q4nx`: an 8-byte +// little-endian header length, that many bytes of JSON mapping tensor name -> {dtype, shape, data_offsets}, then the +// payload. There is no graph and no `model.onnx`, which is why the importer above refuses such a pack before reading a +// byte of it. The tensor NAMES are the model's own HF names, so this reader is also the work list: what it reports is +// what a container-genre plan has to map. +// +// Header only: no weights are read, no device is opened. Exported because it is the half that can be wrong silently +// (an off-by-one header length reports an empty inventory and looks like "a pack with no tensors"). +struct Q4nxTensor { + std::string name; + std::string dtype; + std::vector shape; + int64_t begin = 0; // data_offsets[0] + int64_t end = 0; // data_offsets[1] +}; +struct Q4nxContainer { + int64_t header_bytes = 0; + int64_t payload_bytes = 0; + std::vector tensors; // in header order + std::string why; // non-empty when the container could not be read; tensors is then empty +}; +Q4nxContainer read_q4nx_container(const std::string& path); + // Reads the pack (model.onnx, its external data, both .fconst headers, config.json). // Throws only when the pack is structurally unreadable. Plan plan(const std::string& pack_dir); diff --git a/tests/npu_forward_loader_test.c b/tests/npu_forward_loader_test.c index c32b8513c..fe0b6395e 100644 --- a/tests/npu_forward_loader_test.c +++ b/tests/npu_forward_loader_test.c @@ -165,6 +165,26 @@ int main(void) { model_free(dm); } } + /* engine#1786 (item F1): three of the fifty-eight NPU2 model sources spell the per-layer prefix + * `model.layer.` (singular) where this forward addresses `model.layers.`. model_tensor_by_name() retries the lookup + * with the other spelling AFTER an exact match has failed, and this helper is the whole of that rewrite -- so it + * must swap the leading prefix and nothing else: a global replace would corrupt a tensor that carries the same text + * further along, and a near-miss prefix must be left alone rather than "helpfully" matched. */ + { + char alt[512]; + check(model_tensor_name_alt("model.layers.0.input_layernorm.weight", alt, sizeof alt) == 1 && + strcmp(alt, "model.layer.0.input_layernorm.weight") == 0, + "ALT: model.layers. -> model.layer., the rest untouched"); + check(model_tensor_name_alt("model.layer.0.input_layernorm.weight", alt, sizeof alt) == 1 && + strcmp(alt, "model.layers.0.input_layernorm.weight") == 0, + "ALT: and back, model.layer. -> model.layers."); + check(model_tensor_name_alt("lm_head.weight", alt, sizeof alt) == 0, + "ALT: a model-level tensor has no per-layer spelling to try"); + check(model_tensor_name_alt("model.layersX.0.proj.weight", alt, sizeof alt) == 0, + "ALT: a near-miss prefix is not matched helpfully"); + check(model_tensor_name_alt("model.layers.0.a.weight", alt, 16) == 0, + "ALT: a too-small buffer reports no alternative rather than truncating"); + } char cmd[96]; snprintf(cmd, sizeof cmd, "rm -rf %s", dir); if (system(cmd) != 0) fprintf(stderr, "could not remove %s\n", dir); diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 3d1c246b0..b88cdc0ce 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -4015,6 +4015,107 @@ void test_embedding_f32_component(const std::string& root) { fs::remove_all(b); } +// engine#1786 (item F1): an NPU2 model source is a safetensors container, and this is the reader that decides whether +// a pack is that genre. The failure that matters is the silent one -- a wrong header length yields an empty inventory, +// which reads as "a pack with no tensors" rather than "a malformed header" -- so each arm asserts the reason, not just +// the emptiness. No weights are read and no device is opened: the header is 26,496 B on the smallest real pack. +void test_q4nx_container(const std::string& root) { + namespace fs = std::filesystem; + const fs::path dir = fs::path(root) / "q4nx-container"; + std::error_code ec; + fs::create_directories(dir, ec); + + const auto write_container = [&](const std::string& name, const std::string& header_json, int64_t payload, + bool truncate_header) { + const fs::path p = dir / name; + std::string out(8, '\0'); + int64_t n = int64_t(header_json.size()); + for (int i = 0; i < 8; ++i) out[size_t(i)] = char((n >> (8 * i)) & 0xFF); + out += truncate_header ? header_json.substr(0, header_json.size() / 2) : header_json; + out.append(size_t(payload), '\x22'); + std::ofstream f(p, std::ios::binary); + f.write(out.data(), std::streamsize(out.size())); + return p.string(); + }; + + // (a) the real shape: one BF16 vector and one packed I8 matrix, with the payload the offsets imply. + { + const std::string hdr = + R"({"model.layers.0.input_layernorm.weight":{"dtype":"BF16","shape":[640],"data_offsets":[0,1280]},)" + R"("model.layers.0.mlp.down_proj.weight":{"dtype":"I8","shape":[640,1280],"data_offsets":[1280,820480]}})"; + const auto c = pi::read_q4nx_container(write_container("two.q4nx", hdr, 820480, false)); + check(!c.tensors.empty(), "q4nx container: a two-tensor header reads (" + c.why + ")"); + check(c.tensors.size() == 2, "q4nx container: both tensors are reported"); + check(c.header_bytes == int64_t(hdr.size()), "q4nx container: the 8-byte length is the header's own"); + check(c.payload_bytes == 820480, "q4nx container: the payload is the sum of the data_offsets spans (got " + + std::to_string(c.payload_bytes) + ")"); + bool saw_bf16 = false, saw_i8 = false; + for (const auto& t : c.tensors) { + if (t.dtype == "BF16" && t.shape.size() == 1 && t.shape[0] == 640) saw_bf16 = true; + // The packed matrix: I8 with TWICE the logical element count of a 640-wide FFN is the NPU2 shape. + if (t.dtype == "I8" && t.shape.size() == 2 && t.shape[0] == 640 && t.shape[1] == 1280) saw_i8 = true; + } + check(saw_bf16 && saw_i8, "q4nx container: dtype and shape survive for a vector and a packed matrix"); + } + + // (b) `__metadata__` is a string map, not a tensor, and the vendor's converter does not always write one. + { + const std::string hdr = + R"({"__metadata__":{"group_size":"32"},"t":{"dtype":"I8","shape":[32,256],"data_offsets":[0,5120]}})"; + const auto c = pi::read_q4nx_container(write_container("meta.q4nx", hdr, 5120, false)); + check(c.tensors.size() == 1 && c.tensors[0].name == "t", + "q4nx container: __metadata__ is skipped, not counted as a tensor"); + } + + // (c) a truncated header is named, and reports no tensors rather than a partial inventory. + { + const std::string hdr = R"({"a":{"dtype":"I8","shape":[32,256],"data_offsets":[0,5120]}})"; + const auto c = pi::read_q4nx_container(write_container("short.q4nx", hdr, 5120, true)); + check(c.tensors.empty() && !c.why.empty(), + "q4nx container: a truncated header names the problem instead of reporting no tensors (" + c.why + ")"); + } + + // (e) engine#1786: a pack that IS a container and has NO graph is recognised and reported, not refused at the first + // byte. The alternative -- `npu-pack info` answering a 0.5 GB model source with "cannot open /model.onnx" -- + // describes a missing file on a pack that has no graph by design, and says nothing about what it does contain. + { + const fs::path pdir = dir / "container-genre"; + fs::create_directories(pdir, ec); + const std::string hdr = + R"({"model.embed_tokens.weight":{"dtype":"BF16","shape":[64,8],"data_offsets":[0,1024]},)" + R"("model.layers.0.mlp.down_proj.weight":{"dtype":"I8","shape":[8,16],"data_offsets":[1024,1152]}})"; + write_container("container-genre/model.q4nx", hdr, 1152, false); + const pi::Plan p = pi::plan(pdir.string()); + check(p.container_tensors == 2, "container genre: plan() reports the container's tensors (" + + std::to_string(p.container_tensors) + ")"); + check(p.container_payload_bytes == 1152, "container genre: and the payload the offsets imply"); + check(p.container_dtypes.count("BF16") == 1 && p.container_dtypes.count("I8") == 1, + "container genre: and both dtypes it found"); + // The note must say it is NOT importable: a plan that listed these as importable would be worse than the + // refusal it replaces, because it would claim a compute mapping this importer does not have. + check(p.container_note.find("not importable") != std::string::npos, + "container genre: the note says the pack is recognised and not importable (" + p.container_note + ")"); + const json m = json::parse(p.manifest()); + check(m.contains("container") && m["container"]["tensors"] == 2, + "container genre: and manifest() reports it, so npu-pack info prints it"); + } + + // (d) an implausible length is refused by the number, so a mis-typed length cannot make the caller allocate it. + { + const fs::path p = dir / "huge.q4nx"; + std::string out(8, '\0'); + const int64_t n = int64_t(1) << 40; + for (int i = 0; i < 8; ++i) out[size_t(i)] = char((n >> (8 * i)) & 0xFF); + { + std::ofstream f(p, std::ios::binary); + f.write(out.data(), std::streamsize(out.size())); + } // flushed and closed: the reader must see the bytes, not an empty file + const auto c = pi::read_q4nx_container(p.string()); + check(c.tensors.empty() && c.why.find("implausible") != std::string::npos, + "q4nx container: an implausible header length is refused by the number (" + c.why + ")"); + } +} + // engine#1725: a multi-component embedding carries a PER-LAYER table and its projection norm beside the token table, and // the importer writes both into the container so the forward path can gather them. Measured shapes on @@ -4584,6 +4685,7 @@ int main(int argc, char** argv) { test_fconst_variable_stride(golden_dir.empty() ? "." : golden_dir); test_attn_payload_off(); test_text_config_geometry(golden_dir.empty() ? "." : golden_dir); + test_q4nx_container(golden_dir.empty() ? "." : golden_dir); test_hybrid_pack_import(golden_dir.empty() ? "." : golden_dir); test_rope_parameters_theta(golden_dir.empty() ? "." : golden_dir); test_three_input_matmulnbits(golden_dir.empty() ? "." : golden_dir); diff --git a/tools/flm/q4nx_container_report.py b/tools/flm/q4nx_container_report.py new file mode 100755 index 000000000..0e8243983 --- /dev/null +++ b/tools/flm/q4nx_container_report.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Report a FastFlowLM NPU2 model source's container without loading a device (engine#1786, item F1). + +The NPU2 model sources on the Hub are a safetensors container named `model.q4nx`: 8-byte little-endian +header length, then that many bytes of JSON mapping tensor name -> {dtype, shape, data_offsets}, then +the payload. The tensor NAMES are the model's own HF names, so a report from this script says exactly +what the importer has to accept. No device, no model weights are read: only the header. + + tools/flm/q4nx_container_report.py [...] +""" +import json +import os +import struct +import sys +from collections import Counter + + +def read_header(path): + with open(path, "rb") as f: + n = struct.unpack(" 64 << 20: + raise ValueError(f"{path}: implausible header length {n}") + hdr = json.loads(f.read(n)) + meta = hdr.pop("__metadata__", {}) or {} + return hdr, meta, n + + +def report(path): + hdr, meta, n = read_header(path) + dtypes = Counter(v.get("dtype", "?") for v in hdr.values()) + payload = sum(v["data_offsets"][1] - v["data_offsets"][0] for v in hdr.values()) + layers = {k.split(".layers.")[1].split(".")[0] for k in hdr if ".layers." in k} + print(f"{os.path.basename(path)}: header {n} B, {len(hdr)} tensors, payload {payload/1e9:.3f} GB") + print(f" dtypes {dict(dtypes)}") + print(f" layers seen: {len(layers)} metadata keys: {sorted(meta)}") + if meta: + for k, v in list(meta.items())[:8]: + print(f" {k} = {v}") + # The first layer's tensors, because they are the set the importer must map per layer. + for k in sorted(hdr): + if ".layers.0." in k: + v = hdr[k] + print(f" {k:52} {v['dtype']:5} {v['shape']}") + return 0 + + +def main(argv): + if len(argv) < 2: + print(__doc__.strip().splitlines()[-1], file=sys.stderr) + return 2 + rc = 0 + for arg in argv[1:]: + path = os.path.join(arg, "model.q4nx") if os.path.isdir(arg) else arg + if not os.path.isfile(path): + print(f"{arg}: no model.q4nx", file=sys.stderr) + rc = 1 + continue + rc |= report(path) + return rc + + +if __name__ == "__main__": + sys.exit(main(sys.argv))