Skip to content
Merged
28 changes: 28 additions & 0 deletions npu/forward/include/model.h
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

#include <stdint.h>
#include <stddef.h>
#include <string.h>
#include "common.h"

#ifdef __cplusplus
Expand Down Expand Up @@ -130,6 +131,33 @@ void* model_tensor_data(ModelWeights* mw, TensorDesc* desc);
// Used for architecture-specific tensors the fixed fields do not cover.
// (Distinct from the legacy model_find_tensor(name, mw), which returns an index.)
TensorDesc* model_tensor_by_name(ModelWeights* mw, const char* name);
// engine#1786 (item F1): three FastFlowLM NPU2 model sources spell the per-layer prefix `model.layer.` (singular)
// where this forward addresses `model.layers.`. Writes the OTHER spelling into `out` and returns 1 when `name` carries
// either prefix, 0 when it does not. Only the leading prefix is swapped, once: a caller that wants a fallback retries
// the lookup with this name AFTER an exact match has failed, so a container carrying both spellings still resolves
// exactly first. Pure string work, exported so a test can pin it without a container.
static inline int model_tensor_name_alt(const char* name, char* out, size_t cap) {
static const char kPlural[] = "model.layers.";
static const char kSingular[] = "model.layer.";
const size_t nl = sizeof(kPlural) - 1, ns = sizeof(kSingular) - 1;
const char* to;
size_t fl;
if (!name || !out || cap == 0) return 0;
if (strncmp(name, kPlural, nl) == 0) { /* model.layers. -> model.layer. */
to = kSingular; fl = nl; /* fl is the MATCHED prefix's length, not the target's */
} else if (strncmp(name, kSingular, ns) == 0) { /* model.layer. -> model.layers. */
to = kPlural; fl = ns;
} else {
return 0;
}
const char* rest = name + fl; /* everything after the prefix, unchanged */
const size_t rl = strlen(rest);
const size_t tl = (to == kPlural) ? nl : ns;
if (tl + rl + 1 > cap) return 0;
memcpy(out, to, tl);
memcpy(out + tl, rest, rl + 1);
return 1;
}
void model_print_info(ModelWeights* mw);
int model_find_tensor(const char* name, ModelWeights* mw);

Expand Down
12 changes: 12 additions & 0 deletions npu/forward/src/model.c
Original file line number Diff line number Diff line change
Expand Up @@ -1157,12 +1157,24 @@ void model_free(ModelWeights* mw) {
free(mw);
}

// engine#1786 (item F1): the per-layer prefix, in the two spellings the NPU2 model sources use. `layers.` is this
// forward's own spelling; `layer.` is what three of the fifty-eight containers carry (`model.layer.0.input_layernorm…`),
// where every other tensor matches by name. Swapping one prefix is a spelling fallback, not a rename: it happens only
// after an exact match has failed, so a container that carries both is unaffected.

TensorDesc* model_tensor_by_name(ModelWeights* mw, const char* name) {
if (!mw || !mw->all_tensors || !name) return NULL;
for (int i = 0; i < mw->all_count; i++) {
const char* n = mw->all_tensors[i].name;
if (n[0] && strcmp(n, name) == 0) return &mw->all_tensors[i];
}
// Exact match failed: try the other per-layer spelling before reporting absence (engine#1786).
char alt[512];
if (!model_tensor_name_alt(name, alt, sizeof(alt))) return NULL;
for (int i = 0; i < mw->all_count; i++) {
const char* n = mw->all_tensors[i].name;
if (n[0] && strcmp(n, alt) == 0) return &mw->all_tensors[i];
}
return NULL;
}

Expand Down
114 changes: 114 additions & 0 deletions npu/pack_import.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1704,6 +1704,85 @@ bool fconst_set_block_headers(FconstGeom& g, const std::vector<int64_t>& inner)

std::string find_token_fconst_exported(const std::string& pack_dir) { return find_token_fconst(pack_dir); }

// engine#1786 (item F1): read a FastFlowLM NPU2 model source's container header. Safetensors layout: an 8-byte
// little-endian header length, then that many bytes of JSON. `__metadata__` is a string map, not a tensor, and the
// vendor's converter does not always write one -- so its presence is not a genre test and its absence is not an error.
// Every failure names what was wrong, because the caller's next move ("refuse this pack" vs "read it") depends on which.
Q4nxContainer read_q4nx_container(const std::string& path) {
Q4nxContainer out;
std::ifstream f(path, std::ios::binary);
if (!f) {
out.why = "cannot open " + path;
return out;
}
unsigned char len[8] = {0};
if (!f.read(reinterpret_cast<char*>(len), 8)) {
out.why = path + ": shorter than the 8-byte safetensors header length";
return out;
}
int64_t n = 0;
for (int i = 7; i >= 0; --i) n = (n << 8) | int64_t(len[i]);
// A header this large is not a container: the largest real one seen on the NPU2 sources is 26,496 B (237 tensors).
// Refusing above 64 MiB keeps a mis-typed length from making the caller allocate whatever the file claims.
if (n <= 0 || n > (int64_t(64) << 20)) {
out.why = path + ": implausible header length " + std::to_string(n) + " B";
return out;
}
std::string hdr(size_t(n), '\0');
if (!f.read(&hdr[0], n)) {
out.why = path + ": truncated header (" + std::to_string(n) + " B claimed)";
return out;
}
out.header_bytes = n;
json j;
try {
j = json::parse(hdr);
} catch (const std::exception& e) {
out.why = path + ": header is not JSON (" + e.what() + ")";
return out;
}
if (!j.is_object()) {
out.why = path + ": header is not a JSON object";
return out;
}
for (auto it = j.begin(); it != j.end(); ++it) {
if (it.key() == "__metadata__") continue;
if (!it.value().is_object()) {
out.why = path + ": tensor '" + it.key() + "' is not an object";
out.tensors.clear();
return out;
}
Q4nxTensor t;
t.name = it.key();
const json& v = it.value();
t.dtype = v.value("dtype", std::string());
if (t.dtype.empty()) {
out.why = path + ": tensor '" + t.name + "' states no dtype";
out.tensors.clear();
return out;
}
if (v.contains("shape") && v["shape"].is_array())
for (const json& d : v["shape"]) t.shape.push_back(d.is_number_integer() ? d.get<int64_t>() : 0);
if (!v.contains("data_offsets") || !v["data_offsets"].is_array() || v["data_offsets"].size() != 2 ||
!v["data_offsets"][0].is_number_integer() || !v["data_offsets"][1].is_number_integer()) {
out.why = path + ": tensor '" + t.name + "' has no [begin, end] data_offsets";
out.tensors.clear();
return out;
}
t.begin = v["data_offsets"][0].get<int64_t>();
t.end = v["data_offsets"][1].get<int64_t>();
if (t.begin < 0 || t.end < t.begin) {
out.why = path + ": tensor '" + t.name + "' has data_offsets [" + std::to_string(t.begin) + ", " +
std::to_string(t.end) + "]";
out.tensors.clear();
return out;
}
out.payload_bytes += t.end - t.begin;
out.tensors.push_back(std::move(t));
}
return out;
}

// engine#1638: the geometry READ (not just its dims), so a test can assert where each number came from -- the
// text_config lift and the top-level-wins rule are both properties of load_geometry(), not of read_dims().
json load_geometry_exported(const std::string& pack_dir, std::string* source) { return load_geometry(pack_dir, source); }
Expand Down Expand Up @@ -2840,6 +2919,18 @@ std::string Plan::manifest() const {
{"rule", "length == N * (K/128 + 1) * 68"}};
j["graph_weights"] = g;
}
// engine#1786 (item F1): a FastFlowLM NPU2 model source has no graph, so the census above is absent and this is
// what `info` reports instead -- a count and a dtype map taken from the container's own header, plus what the genre
// still needs. Measured on FastFlowLM/Gemma3-270M-NPU2 (0.54 GB): 237 tensors, 0.503 GB, {BF16: 110, I8: 127}.
if (container_tensors >= 0) {
json c;
c["format"] = "safetensors (model.q4nx)";
c["tensors"] = container_tensors;
c["payload_bytes"] = container_payload_bytes;
c["dtypes"] = container_dtypes;
c["note"] = container_note;
j["container"] = c;
}
j["mlp_section"] = {{"gate_up_units_measured", mlp_gu_units},
{"down_groups_measured", mlp_down_groups},
{"note", mlp_note}};
Expand Down Expand Up @@ -2891,6 +2982,29 @@ Plan plan(const std::string& pack_dir) {
p.dir = pack_dir;
p.pack = fs::path(pack_dir).lexically_normal().filename().string();

// engine#1786 (item F1): the FastFlowLM NPU2 model sources are a safetensors container named model.q4nx with no
// graph at all. Detect that genre before anything looks for a graph, report what the container holds, and say what
// it still needs -- the alternative is the first byte failing with "cannot open <pack>/model.onnx", which describes
// a missing file on a pack that has no graph by design.
{
std::error_code ec;
const std::string container = pack_dir + "/model.q4nx";
const std::string graph_now = graph_filename(pack_dir);
if (fs::exists(container, ec) && !fs::exists(pack_dir + "/" + graph_now, ec)) {
const Q4nxContainer c = read_q4nx_container(container);
p.container_tensors = int64_t(c.tensors.size());
p.container_payload_bytes = c.payload_bytes;
for (const auto& t : c.tensors) p.container_dtypes[t.dtype]++;
p.container_note = c.why.empty()
? "the pack is a safetensors container (model.q4nx): " + std::to_string(c.tensors.size()) +
" tensors, " + std::to_string(c.payload_bytes) + " B of payload, no graph. Reading it as a"
" weights source needs the container-genre plan (engine#1786, item F1); until then this pack is"
" recognised and reported, not importable."
: "model.q4nx is present but unreadable: " + c.why;
return p;
}
}

std::string geom_src;
json geom = load_geometry(pack_dir, &geom_src);
// `intermediate_size` is the one key no 1.6/1.7 pack states: genai_config.json has no such field, and on the
Expand Down
31 changes: 31 additions & 0 deletions npu/pack_import.h
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,14 @@ struct Plan {
// third-party parser: the rai_1.3/1.6/1.7.1 line ships its 4-bit weights as ordinary ONNX initializers, while
// 1.8 and R1-Distill ship placeholders and keep the weights in model.bin (pi-inferzo, 2026-11-08).
// graph_matmuls < 0 means "no graph was read", which is why it is not just a count.
// engine#1786 (item F1): a FastFlowLM NPU2 model source is a safetensors container (`model.q4nx`) with no graph,
// so the two census fields above stay -1. `container_tensors` is -1 for every pack that is not that genre; when it
// is not, it is the container's tensor count and the dtype map says what the reader found. Reported rather than
// refused at the first byte, because "cannot open model.onnx" says nothing about a pack that has no graph by design.
int64_t container_tensors = -1;
int64_t container_payload_bytes = 0;
std::map<std::string, int64_t> container_dtypes;
std::string container_note; // what the container genre still needs, empty for a non-container pack
int64_t graph_matmuls = -1; // MatMulNBits nodes in the main graph
int64_t graph_real_weights = 0; // of those, how many point at an initializer with dims (i.e. real weights)
int64_t graph_external = 0; // initializers that declare external data (location + offset)
Expand Down Expand Up @@ -251,6 +259,29 @@ int intermediate_size_from_graph(const std::string& model_path, int64_t hidden);
// multi-component one (`type: compositemodel`), whose geometry files stay at the root. Exported for the unit test.
std::string find_token_fconst_exported(const std::string& pack_dir);

// A FastFlowLM NPU2 model source (engine#1786, item F1) is a safetensors container named `model.q4nx`: an 8-byte
// little-endian header length, that many bytes of JSON mapping tensor name -> {dtype, shape, data_offsets}, then the
// payload. There is no graph and no `model.onnx`, which is why the importer above refuses such a pack before reading a
// byte of it. The tensor NAMES are the model's own HF names, so this reader is also the work list: what it reports is
// what a container-genre plan has to map.
//
// Header only: no weights are read, no device is opened. Exported because it is the half that can be wrong silently
// (an off-by-one header length reports an empty inventory and looks like "a pack with no tensors").
struct Q4nxTensor {
std::string name;
std::string dtype;
std::vector<int64_t> shape;
int64_t begin = 0; // data_offsets[0]
int64_t end = 0; // data_offsets[1]
};
struct Q4nxContainer {
int64_t header_bytes = 0;
int64_t payload_bytes = 0;
std::vector<Q4nxTensor> tensors; // in header order
std::string why; // non-empty when the container could not be read; tensors is then empty
};
Q4nxContainer read_q4nx_container(const std::string& path);

// Reads the pack (model.onnx, its external data, both .fconst headers, config.json).
// Throws only when the pack is structurally unreadable.
Plan plan(const std::string& pack_dir);
Expand Down
20 changes: 20 additions & 0 deletions tests/npu_forward_loader_test.c
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,26 @@ int main(void) {
model_free(dm);
}
}
/* engine#1786 (item F1): three of the fifty-eight NPU2 model sources spell the per-layer prefix
* `model.layer.` (singular) where this forward addresses `model.layers.`. model_tensor_by_name() retries the lookup
* with the other spelling AFTER an exact match has failed, and this helper is the whole of that rewrite -- so it
* must swap the leading prefix and nothing else: a global replace would corrupt a tensor that carries the same text
* further along, and a near-miss prefix must be left alone rather than "helpfully" matched. */
{
char alt[512];
check(model_tensor_name_alt("model.layers.0.input_layernorm.weight", alt, sizeof alt) == 1 &&
strcmp(alt, "model.layer.0.input_layernorm.weight") == 0,
"ALT: model.layers. -> model.layer., the rest untouched");
check(model_tensor_name_alt("model.layer.0.input_layernorm.weight", alt, sizeof alt) == 1 &&
strcmp(alt, "model.layers.0.input_layernorm.weight") == 0,
"ALT: and back, model.layer. -> model.layers.");
check(model_tensor_name_alt("lm_head.weight", alt, sizeof alt) == 0,
"ALT: a model-level tensor has no per-layer spelling to try");
check(model_tensor_name_alt("model.layersX.0.proj.weight", alt, sizeof alt) == 0,
"ALT: a near-miss prefix is not matched helpfully");
check(model_tensor_name_alt("model.layers.0.a.weight", alt, 16) == 0,
"ALT: a too-small buffer reports no alternative rather than truncating");
}
char cmd[96];
snprintf(cmd, sizeof cmd, "rm -rf %s", dir);
if (system(cmd) != 0) fprintf(stderr, "could not remove %s\n", dir);
Expand Down
Loading
Loading