From 03a3066b7d1b3600a9fb545e59acb713db2b892c Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 12 Nov 2026 01:25:04 -0400 Subject: [PATCH 1/3] npu: the per-op forward reads LFM3, so LFM3-1.7B-NPU2 loads and answers (Refs #1786) FastFlowLM's LFM3-1.7B-NPU2 was refused at load by name ("conv" in layer_types). LFM3 is LFM2's conv/attention hybrid plus four rules from LiquidAI's modeling_lfm3.py, all now run: - layer_types "conv" (exact match), cross-checked layer for layer against the container's conv tensors; - use_sandwich_norm: norms on the block output and the MLP output, conv layers included (extra_layer_norms 2); - a per-head sigmoid output gate, self_attn.gate_proj [heads, hidden], before o_proj (step and batch); - zero-tail partial RoPE on the full layers (Gemma4's proportional rule). model_load renames the upstream tensor names the pack keeps (conv.*, operator_norm, ffn_norm, feed_forward.w1/w2/w3, post_operator_norm, post_ffn_norm, embedding_norm) to the canonical ones, only where the canonical name is absent. Host check against a numpy transcription of modeling_lfm3.py on the vendor QAD-Q4_0 GGUF's weights: last-position logits corr 0.9995, same top 10. batch_check_state on the pack: rel-L1 0. New: q4nx_lfm3_semantics_test (20 checks) and batch_check_state_lfm3 (fabricated LFM3 under the upstream names); dropping the batched gate fails it (rel-L1 0.46), dropping the "conv" kind fails it. Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 13 +++ docs/npu.md | 31 ++++++ npu/forward/include/model.h | 9 ++ npu/forward/include/q4nx_forward.h | 4 + npu/forward/include/q4nx_semantics.h | 15 ++- npu/forward/src/model.c | 58 ++++++++++ npu/forward/src/q4nx_forward.cpp | 45 ++++++++ npu/forward/src/q4nx_semantics.c | 26 ++++- tests/batch_check_state_ci.py | 46 +++++++- tests/q4nx_lfm3_semantics_test.cpp | 161 +++++++++++++++++++++++++++ 10 files changed, 395 insertions(+), 13 deletions(-) create mode 100644 tests/q4nx_lfm3_semantics_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 73db2e28d..f4427115c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1748,6 +1748,14 @@ target_include_directories(q4nx_gemma3_semantics_test PRIVATE npu/forward/includ target_link_libraries(q4nx_gemma3_semantics_test PRIVATE m) add_test(NAME q4nx_gemma3_semantics COMMAND q4nx_gemma3_semantics_test) add_dependencies(npu_sampling_test q4nx_gemma3_semantics_test) +# LFM3 (engine#1786): "conv" in layer_types, the sandwich, the per-head gate, the zero-tail RoPE, and model_load's +# renaming of the upstream tensor names. Links the two C sources directly, no model and no device. +add_executable(q4nx_lfm3_semantics_test tests/q4nx_lfm3_semantics_test.cpp npu/forward/src/model.c + npu/forward/src/q4nx_semantics.c) +target_include_directories(q4nx_lfm3_semantics_test PRIVATE npu/forward/include) +target_link_libraries(q4nx_lfm3_semantics_test PRIVATE m) +add_test(NAME q4nx_lfm3_semantics COMMAND q4nx_lfm3_semantics_test) +add_dependencies(npu_sampling_test q4nx_lfm3_semantics_test) add_dependencies(npu_sampling_test q4nx_qk_norm_test) add_executable(q4nx_layer_kinds_test tests/q4nx_layer_kinds_test.c npu/forward/src/q4nx_semantics.c) target_include_directories(q4nx_layer_kinds_test PRIVATE npu/forward/include) @@ -2115,6 +2123,11 @@ if(ONEBIT_NPU_HOST AND NOT ONEBIT_NPU AND NOT WIN32 AND Python3_Interpreter_FOUN COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/batch_check_state_ci.py $ lfm2) set_tests_properties(batch_check_state_lfm2 PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 120) + # LFM3 (engine#1786): the upstream tensor names, "conv" in layer_types, sandwich norms and the per-head gate. + add_test(NAME batch_check_state_lfm3 + COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/batch_check_state_ci.py + $ lfm3) + set_tests_properties(batch_check_state_lfm3 PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 120) # A fabricated qwen3_5 hybrid (a linear-attention layer, then attention), off the kernel's N grid on purpose. add_test(NAME batch_check_state_qwen35 COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/batch_check_state_ci.py diff --git a/docs/npu.md b/docs/npu.md index 63c28eb7c..e9d736432 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1998,6 +1998,37 @@ and against the pack's own embedding (`lm_head`: 0.9963, at most 0.57 of a step the known-answer prompt `Paris` and explains Rayleigh scattering in coherent sentences. Before, it failed at load with `chunk 4266 B is not a kind`. No kernel set for its shapes was built or run (the NPU is off). +**LFM3-1.7B-NPU2: the forward reads LFM3 (engine#1786).** LiquidAI's LFM3 is LFM2's hybrid of gated short +convolutions and attention, plus four rules from its `modeling_lfm3.py`. The forward now runs all four: + +- `layer_types` names the conv layers `"conv"`, beside `full_attention` and `sliding_attention`. Each `"conv"` entry + has to match a layer whose conv tensors are in the container, and a mismatch either way is refused by name. + `"conv"` is an exact match: `conv_attention` is still an unknown kind. +- `use_sandwich_norm`: a norm on the conv or attention output before the residual add, and one on the MLP output. + It is the same pair the Gemma3 path runs (`extra_layer_norms` 2), and here the conv layers take it too. +- A per-head output gate. Every attention layer has `self_attn.gate_proj` `[heads, hidden]` (bf16 in the pack), and + each head's context is multiplied by `sigmoid(gate_proj . x)` before `o_proj`, `x` being the block's normed input. +- Zero-tail partial RoPE on the full layers. With `partial_rotary_factor` 0.5 the first 32 frequencies follow the + progression of the whole 128-channel head and the other 32 are zero, with `rotate_half` pairing across the whole + head. That is the rule Gemma4's "proportional" RoPE already had. The vendor GGUF writes the same rule as + `rope_freqs` (1 for 32 factors, 1e30 for the rest). + +The pack keeps the upstream tensor names (`conv.*`, `operator_norm`, `ffn_norm`, `feed_forward.w1/w2/w3`, +`post_operator_norm`, `post_ffn_norm`, `embedding_norm`). `model_load` renames them to the canonical ones, and only +where the canonical name is absent (`model_alias_lfm_names`, `model.h`). `ffn_norm` is the MLP input's norm, so it +becomes `pre_feedforward_layernorm` in a sandwich layer and `post_attention_layernorm` otherwise. + +Checked on the host, 2026-11-12, against a numpy transcription of `modeling_lfm3.py`. Run on the bf16 checkpoint, +the reference's last-position logits correlate 0.939 with ours. Run on the dequantized weights of the vendor's +`LFM3-1.7B-QAD-Q4_0` GGUF, which this pack's I8 tensors come from, they correlate 0.9995, with the same top 10. +The two references differ from each other by the same 0.94, so that gap is the quantization-aware weights, not the +arithmetic. Each rule matters on this prompt: with the gate off the bf16 reference falls to 0.907 against ours, +with the usual partial RoPE to 0.822 (and its top token changes), and without the sandwich to 0.134. +`batch_check_state_test` on the pack: batched equals sequential, rel-L1 0. The known-answer prompt, with thinking +off (the template's ``), gives `ParisParis<|im_end|>`. The reference makes the same three +choices: `` after the first `Paris`, then `Paris` again. Asked why the sky is blue, it explains Rayleigh +scattering in correct sentences (80 tokens, greedy). No kernel set for its shapes was built or run (the NPU is off). + **Per-head q/k normalisation is read from the container (engine#1788).** Whether the forward normalises q and k per head used to be decided by `config.json`'s `model_type` alone: on for `qwen3*` and `gemma*`. A pack of the same architecture under another name got it wrong in silence. FastFlowLM's MiniCPM-V 4.7 diff --git a/npu/forward/include/model.h b/npu/forward/include/model.h index eafe4c23b..1e06af792 100644 --- a/npu/forward/include/model.h +++ b/npu/forward/include/model.h @@ -131,6 +131,15 @@ void* model_tensor_data(ModelWeights* mw, TensorDesc* desc); // Used for architecture-specific tensors the fixed fields do not cover. // (Distinct from the legacy model_find_tensor(name, mw), which returns an index.) TensorDesc* model_tensor_by_name(ModelWeights* mw, const char* name); + +// LFM3 packs (FastFlowLM/LFM3-1.7B-NPU2) keep the upstream Liquid names -- conv.*, operator_norm, ffn_norm, +// feed_forward.w1/w2/w3, post_operator_norm, post_ffn_norm, embedding_norm -- where the LFM2 packs carry the +// canonical ones (shortconv.*, input_layernorm, mlp.*, post_attention_layernorm, model.norm). This renames each +// to the canonical name, in place, and only where that name is not already taken. ffn_norm is the norm in front +// of the MLP, so it becomes pre_feedforward_layernorm in a layer that also has post_operator_norm (the sandwich +// form, where post_attention_layernorm is the attention/conv output's norm) and post_attention_layernorm +// otherwise. Returns the number of tensors renamed. +int model_alias_lfm_names(TensorDesc* tensors, int count); // engine#1786 (item F1): three FastFlowLM NPU2 model sources spell the per-layer prefix `model.layer.` (singular) // where this forward addresses `model.layers.`. Writes the OTHER spelling into `out` and returns 1 when `name` carries // either prefix, 0 when it does not. Only the leading prefix is swapped, once: a caller that wants a fallback retries diff --git a/npu/forward/include/q4nx_forward.h b/npu/forward/include/q4nx_forward.h index 5ec276cff..f43e91fd9 100644 --- a/npu/forward/include/q4nx_forward.h +++ b/npu/forward/include/q4nx_forward.h @@ -647,6 +647,10 @@ class Q4nxNpuForward { // Layout and maths: docs/qwen35-4b-weight-format.md. int lin_kd_ = 0, lin_vd_ = 0; // key_dim = nkh*khd, value_dim = nvh*vhd int attn_gate_ = 0; // qwen3_5 attn_output_gate + // LFM3's per-head output gate, [NH][H] per attention layer (empty for conv layers and other models): + // ctx of head h *= sigmoid(head_gate_[l][h] . xn), xn the operator-normed layer input. + std::vector> head_gate_; + void apply_head_gate(int l, const float* xn, float* ctx, int hd) const; int lin_nkh_ = 0, lin_nvh_ = 0; // head counts int lin_khd_ = 0, lin_vhd_ = 0; // head dims int lin_ck_ = 4; // depthwise conv kernel diff --git a/npu/forward/include/q4nx_semantics.h b/npu/forward/include/q4nx_semantics.h index e7bb68e1f..bc04f6602 100644 --- a/npu/forward/include/q4nx_semantics.h +++ b/npu/forward/include/q4nx_semantics.h @@ -42,12 +42,15 @@ typedef enum { /* Attention layer kinds, from layer_types. FULL and SLIDING run the attention block, LINEAR runs the * GDN recurrence (lin_kd_/conv_state_); OTHER is anything else, which nothing implements - the forward - * refuses a container whose layer_types contains one, by name (unknown_layer_type). */ + * refuses a container whose layer_types contains one, by name (unknown_layer_type). CONV is LFM3's + * spelling of LFM2's gated short convolution ("conv"); the forward runs it only where the container holds + * that layer's conv tensors, and refuses a mismatch either way. */ typedef enum { Q4NX_LAYER_FULL = 0, Q4NX_LAYER_SLIDING, Q4NX_LAYER_LINEAR, - Q4NX_LAYER_OTHER + Q4NX_LAYER_OTHER, + Q4NX_LAYER_CONV } Q4nxLayerType; typedef struct { @@ -153,8 +156,14 @@ typedef struct { int num_kv_shared_layers; /* Gemma4 adds pre/post feedforward norms and a post-PLE norm that the plain * layer does not have (1). Gemma3 has the same sandwich around attention and - * MLP but no post-PLE norm and no layer scalar (2). Source: model_type. */ + * MLP but no post-PLE norm and no layer scalar (2). LFM3's use_sandwich_norm is the same pair of norms + * around both blocks (post_operator_norm, post_ffn_norm), so it is 2 as well. Source: model_type, and + * use_sandwich_norm for lfm3 (default true, as configuration_lfm3.py). */ int extra_layer_norms; + /* LFM3: every attention layer carries self_attn.gate_proj [heads, hidden] (bf16), and each head's + * context is multiplied by sigmoid(gate_proj . x) before o_proj, x being the operator-normed input. + * Source: model_type. */ + int attn_head_gate; /* Gemma3 states the sliding layers' RoPE base on its own, as * `rope_local_base_freq`, and its `rope_scaling` applies to the full layers * only. 1 when that key was read into rope_theta_sliding. */ diff --git a/npu/forward/src/model.c b/npu/forward/src/model.c index 21a3d119c..b65b67383 100644 --- a/npu/forward/src/model.c +++ b/npu/forward/src/model.c @@ -850,6 +850,10 @@ ModelWeights* model_load(const char* path, ModelConfig config) { } LOG_INFO("Found %d tensors in metadata", num_tensors); + { + const int renamed = model_alias_lfm_names(tensors, num_tensors); + if (renamed > 0) LOG_INFO("Renamed %d LFM tensor names to the canonical ones", renamed); + } // engine#1185: a container imported from an OGA pack stores its I8 weights as 6,144-byte q4_1f // chunks (fp32 scale/zero) instead of the 5,120-byte q4_1 the kernels here dequantise in-kernel. @@ -1162,6 +1166,60 @@ void model_free(ModelWeights* mw) { // where every other tensor matches by name. Swapping one prefix is a spelling fallback, not a rename: it happens only // after an exact match has failed, so a container that carries both is unaffected. +static int name_taken(const TensorDesc* t, int count, const char* name) { + for (int i = 0; i < count; i++) + if (strcmp(t[i].name, name) == 0) return 1; + return 0; +} + +int model_alias_lfm_names(TensorDesc* t, int count) { + static const char* const kLayerAlias[][2] = { + {"conv.in_proj.weight", "shortconv.in_proj.weight"}, + {"conv.conv.weight", "shortconv.conv.weight"}, + {"conv.out_proj.weight", "shortconv.out_proj.weight"}, + {"operator_norm.weight", "input_layernorm.weight"}, + {"feed_forward.w1.weight", "mlp.gate_proj.weight"}, + {"feed_forward.w3.weight", "mlp.up_proj.weight"}, + {"feed_forward.w2.weight", "mlp.down_proj.weight"}, + {"post_operator_norm.weight", "post_attention_layernorm.weight"}, + {"post_ffn_norm.weight", "post_feedforward_layernorm.weight"}, + }; + if (!t || count <= 0) return 0; + int renamed = 0; + char want[sizeof(t[0].name)]; + for (int i = 0; i < count; i++) { + int l = -1, off = 0; + if (sscanf(t[i].name, "model.layers.%d.%n", &l, &off) != 1 || off <= 0) { + if (strcmp(t[i].name, "model.embedding_norm.weight") == 0 && + !name_taken(t, count, "model.norm.weight")) { + snprintf(t[i].name, sizeof(t[i].name), "%s", "model.norm.weight"); + renamed++; + } + continue; + } + const char* suffix = t[i].name + off; + const char* to = NULL; + if (strcmp(suffix, "ffn_norm.weight") == 0) { + // decided from the layer's own tensors: the sandwich form also has post_operator_norm + char probe[sizeof(t[0].name)]; + snprintf(probe, sizeof(probe), "model.layers.%d.post_operator_norm.weight", l); + const int sandwich = name_taken(t, count, probe); + snprintf(probe, sizeof(probe), "model.layers.%d.post_attention_layernorm.weight", l); + to = (sandwich || name_taken(t, count, probe)) ? "pre_feedforward_layernorm.weight" + : "post_attention_layernorm.weight"; + } else { + for (size_t a = 0; a < sizeof(kLayerAlias) / sizeof(kLayerAlias[0]); a++) + if (strcmp(suffix, kLayerAlias[a][0]) == 0) { to = kLayerAlias[a][1]; break; } + } + if (!to) continue; + snprintf(want, sizeof(want), "model.layers.%d.%s", l, to); + if (name_taken(t, count, want)) continue; + snprintf(t[i].name, sizeof(t[i].name), "%s", want); + renamed++; + } + return renamed; +} + TensorDesc* model_tensor_by_name(ModelWeights* mw, const char* name) { if (!mw || !mw->all_tensors || !name) return NULL; for (int i = 0; i < mw->all_count; i++) { diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 494562591..d06ee16bc 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -1112,6 +1112,19 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, "position; file the kind you need instead of converting it"; return false; } + // LFM3 names its gated short-convolution layers in layer_types ("conv"); LFM2 does not, and the container + // decides there (model_shortconv_layers, above). When both speak they have to agree layer for layer: a + // "conv" entry the container has no conv tensors for would run as attention with no attention weights, + // and a conv layer listed as attention would be given a KV cache and the wrong RoPE. + for (int l = 0; l < sem_.n_layer_types && l < c_.NL; l++) { + const bool said_conv = sem_.layer_types[l] == Q4NX_LAYER_CONV; + if (said_conv == is_conv_layer(l) || (!said_conv && !shortconv_)) continue; + err_ = "config.json layer_types[" + std::to_string(l) + "] is \"" + + (said_conv ? std::string("conv") : std::string("attention")) + "\" but model.q4nx " + + (is_conv_layer(l) ? "holds" : "has no") + " model.layers." + std::to_string(l) + + ".shortconv (conv) tensors for that layer"; + return false; + } // Refuse rather than silently apply full-attention math to a model whose // config declares layer types (or RoPE variants) this forward lacks. // Refusal is now precise. Only layers whose attention MATH we do not @@ -1828,6 +1841,23 @@ bool Q4nxNpuForward::load_norms() { layer_attn_dims(l, hd_l, nkv_l); if (!read_bf16(&mw_->layers[l].input_layernorm_weight, in_norm_[l].data(), H)) return false; if (!read_bf16(&mw_->layers[l].post_attention_layernorm_weight, post_norm_[l].data(), H)) return false; + // LFM3: the per-head sigmoid output gate, self_attn.gate_proj [heads, hidden] in bf16, on every attention + // layer. Missing, it would leave the context ungated, so a layer without one is refused by name. + if (sem_.attn_head_gate) { + if (l == 0) head_gate_.assign((size_t)NL, std::vector()); + const TensorDesc* gd = &mw_->layers[l].self_attn_gate_proj_weight; + if (!is_conv_layer(l)) { + if (absent(gd) || strcmp(gd->dtype, "BF16") != 0 || gd->ndim != 2 || gd->shape[0] != c_.NH || + gd->shape[1] != H) { + err_ = "config.json's model_type gates each attention head (LFM3) and model.layers." + + std::to_string(l) + ".self_attn.gate_proj.weight is not bf16 [num_attention_heads, " + "hidden_size]"; + return false; + } + head_gate_[(size_t)l].resize((size_t)c_.NH * (size_t)H); + if (!read_bf16(gd, head_gate_[(size_t)l].data(), c_.NH * H)) return false; + } + } // engine#1889: the config says q and k are normalised and this layer has no gain to normalise with. read_bf16 // would hand back zeros for the absent tensor and rmsnorm(q, 0) is 0 for every head, so the layer would attend // uniformly and answer fluent nonsense. A Hunyuan container imported before the importer claimed the gains is @@ -1923,6 +1953,18 @@ bool Q4nxNpuForward::load_norms() { return true; } +void Q4nxNpuForward::apply_head_gate(int l, const float* xn, float* ctx, int hd) const { + if (l < 0 || (size_t)l >= head_gate_.size() || head_gate_[(size_t)l].empty()) return; + const int H = c_.H; + const float* w = head_gate_[(size_t)l].data(); + for (int h = 0; h < c_.NH; h++) { + double z = 0.0; + for (int k = 0; k < H; k++) z += (double)w[(size_t)h * H + k] * (double)xn[k]; + const float g = 1.0f / (1.0f + std::exp(-(float)z)); + for (int i = 0; i < hd; i++) ctx[(size_t)h * hd + i] *= g; + } +} + void Q4nxNpuForward::layer_attn_dims(int l, int& hd, int& nkv) const { const int lt = (l < sem_.n_layer_types) ? sem_.layer_types[l] : Q4NX_LAYER_FULL; if (lt != Q4NX_LAYER_SLIDING && sem_.global_head_dim > 0) { @@ -4316,6 +4358,8 @@ bool Q4nxNpuForward::step_batch_impl(const std::vector& tokens, int start_p for (int m = 0; m < M; m++) if (!row_attn(m, CTX.data() + (size_t)m * ctx_w, true, 0)) return false; } + for (int m = 0; m < M; m++) + apply_head_gate(l, XN.data() + (size_t)m * H, CTX.data() + (size_t)m * ctx_w, l_hd); const Q4nxDesignGeom* go = &designs_[i_o_l]; Q4nxProjections po{}; po.o = wo_.data(); @@ -5493,6 +5537,7 @@ bool Q4nxNpuForward::step_impl(int token, int pos, std::vector& logits, i if (gate_ptr) for (int i = 0; i < nhd; i++) ctx[i] *= 1.0f / (1.0f + expf(-gate_ptr[i])); + apply_head_gate(l, xn.data(), ctx.data(), l_hd); } { const Q4nxDesignGeom* g = &designs_[i_o_l]; diff --git a/npu/forward/src/q4nx_semantics.c b/npu/forward/src/q4nx_semantics.c index 671069218..01702acfb 100644 --- a/npu/forward/src/q4nx_semantics.c +++ b/npu/forward/src/q4nx_semantics.c @@ -229,9 +229,10 @@ int q4nx_semantics_from_config(const char* json_path, const char* model_dir, /* LFM2 / LFM2.5 (Lfm2ForCausalLM): its attention layers carry q_norm/k_norm, applied before RoPE */ const int is_lfm2 = starts_with(model_type, "lfm2") || starts_with(arch, "Lfm2"); + const int is_lfm3 = starts_with(model_type, "lfm3") || starts_with(arch, "Lfm3"); /* per-head q/k RMSNorm: qwen3*, gemma* and lfm2 have q_norm/k_norm; llama/phi do not */ - s->qk_norm = (is_qwen3 || is_gemma || is_lfm2) ? 1 : 0; + s->qk_norm = (is_qwen3 || is_gemma || is_lfm2 || is_lfm3) ? 1 : 0; /* engine#1889: a config that STATES `use_qk_norm` decides it, whatever the name table says. The table is a list * of the families measured so far, so a family it has never heard of got 0 silently: Hunyuan-7B is * `hunyuan_v1_dense` / `HunYuanDenseV1ForCausalLM` with `use_qk_norm: true`, its container carries @@ -408,6 +409,13 @@ int q4nx_semantics_from_config(const char* json_path, const char* model_dir, s->v_norm = is_gemma4 ? 1 : 0; s->proportional_rope = (is_gemma4 && s->per_layer_rope && strstr(json, "\"proportional\"") != NULL) ? 1 : 0; + /* LFM3's partial RoPE is the same rule under rope_type "default" (modeling_lfm3.py, + * compute_default_rope_parameters): the first partial*head_dim//2 frequencies follow the FULL head_dim's + * progression, the rest are zero, and rotate_half pairs across the whole head. The usual partial form + * (pairs inside the first rotary_dim channels, frequencies over rotary_dim) differs from it at every + * rotated pair. */ + if (is_lfm3) s->proportional_rope = 1; + s->attn_head_gate = is_lfm3 ? 1 : 0; /* LM head: tied to the token embedding (the common case) or a SEPARATE * lm_head.weight (Nanbeige4.1-3B sets tie_word_embeddings = false). */ @@ -441,6 +449,7 @@ int q4nx_semantics_from_config(const char* json_path, const char* model_dir, if (strcmp(nm, "full_attention") == 0) kind = Q4NX_LAYER_FULL; else if (strcmp(nm, "sliding_attention") == 0) kind = Q4NX_LAYER_SLIDING; else if (strcmp(nm, "linear_attention") == 0) kind = Q4NX_LAYER_LINEAR; + else if (strcmp(nm, "conv") == 0) kind = Q4NX_LAYER_CONV; else if (!s->unknown_layer_type[0]) { /* Remember the spelling: the forward refuses this container by name rather than serving an * unimplemented layer kind as full attention (engine#1087's shape). */ @@ -523,6 +532,9 @@ int q4nx_semantics_from_config(const char* json_path, const char* model_dir, s->num_kv_shared_layers = (int)(d + 0.5); /* Gemma4 adds pre/post feedforward norms and a post-PLE norm. */ s->extra_layer_norms = is_gemma4 ? 1 : (is_gemma3 ? 2 : 0); + if (is_lfm3) + s->extra_layer_norms = (strstr(json, "use_sandwich_norm\":false") != NULL || + strstr(json, "use_sandwich_norm\": false") != NULL) ? 0 : 2; /* `use_double_wide_mlp` is a JSON boolean, so read it as "present in text". */ s->double_wide_mlp = (strstr(json, "use_double_wide_mlp\":true") != NULL || strstr(json, "use_double_wide_mlp\": true") != NULL) ? 1 : 0; @@ -571,18 +583,22 @@ int q4nx_semantics_linear_layers(const Q4nxSemantics* s) { void q4nx_semantics_layer_mix(const Q4nxSemantics* s, char* out, size_t out_sz) { if (!out || out_sz == 0) return; - int full = 0, slid = 0, lin = 0, oth = 0; + int full = 0, slid = 0, lin = 0, conv = 0, oth = 0; const int n = s ? s->n_layer_types : 0; for (int i = 0; i < n; i++) { switch (s->layer_types[i]) { case Q4NX_LAYER_FULL: full++; break; case Q4NX_LAYER_SLIDING: slid++; break; case Q4NX_LAYER_LINEAR: lin++; break; + case Q4NX_LAYER_CONV: conv++; break; default: oth++; break; } } if (n == 0) { snprintf(out, out_sz, "no layer_types (all full attention)"); return; } - snprintf(out, out_sz, "full=%d sliding=%d linear=%d other=%d", full, slid, lin, oth); + if (conv) + snprintf(out, out_sz, "full=%d sliding=%d linear=%d conv=%d other=%d", full, slid, lin, conv, oth); + else + snprintf(out, out_sz, "full=%d sliding=%d linear=%d other=%d", full, slid, lin, oth); } int q4nx_semantics_qk_norm_from_tensors(Q4nxSemantics* s, const unsigned char* has_q, const unsigned char* has_k, int layers) { @@ -640,10 +656,10 @@ void q4nx_semantics_dump(const Q4nxSemantics* s) { s->rotary_dim_sliding, s->rope_local_base ? ", rope_local_base_freq" : "", s->rope_type_named, s->query_pre_attn_scalar); fprintf(stderr, " [semantics] ple=%d (dim=%d vocab=%d) kv_shared_layers=%d " - "extra_layer_norms=%d double_wide_mlp=%d\n", + "extra_layer_norms=%d double_wide_mlp=%d attn_head_gate=%d\n", q4nx_semantics_uses_ple(s), s->hidden_size_per_layer_input, s->vocab_size_per_layer_input, s->num_kv_shared_layers, - s->extra_layer_norms, s->double_wide_mlp); + s->extra_layer_norms, s->double_wide_mlp, s->attn_head_gate); } } diff --git a/tests/batch_check_state_ci.py b/tests/batch_check_state_ci.py index db9a612c6..0a3bdbaef 100755 --- a/tests/batch_check_state_ci.py +++ b/tests/batch_check_state_ci.py @@ -15,7 +15,7 @@ # limitations under the License. """The batch-check state pin (tests/batch_check_state_test.cpp, engine#1905/#1946) on a container made here. -usage: tests/batch_check_state_ci.py [lfm2|qwen3_5] (needs numpy) +usage: tests/batch_check_state_ci.py [lfm2|lfm3|qwen3_5] (needs numpy) batch_check_state_test needs a Q4NX container, and CI has none, so on its own it skips (77) there and a regression of #1909 -- the NPU_BATCH_CHECK oracle resetting the prefix it replays from -- lands green. This writes a two-layer qwen3 @@ -33,6 +33,13 @@ pack at layer 0, a conv layer with no attention, and the suite was green with that in it), and a batched prefill of the conv layers leaves the logits and the state a token-by-token run leaves, which is what the test binary compares. +With `lfm3` it is an LFM3-shaped hybrid (engine#1786): conv, full attention, conv, sliding attention (window 2), +written under the UPSTREAM Liquid names (conv.*, operator_norm, ffn_norm, feed_forward.w1/w2/w3, post_operator_norm, +post_ffn_norm, embedding_norm) as FastFlowLM's LFM3-1.7B-NPU2 pack keeps them, with `layer_types` naming "conv", +the zero-tail partial RoPE on the full layer, the sandwich norms, and a bf16 per-head output gate +(self_attn.gate_proj [heads, hidden]). What it pins: model_load's renaming, the forward accepting "conv" in +layer_types, and the batched prefill running the gate and the sandwich exactly as the token-by-token path does. + With `qwen3_5` it is a qwen3_5 hybrid (engine#1889): a linear-attention (GDN) layer, then a full-attention layer with q/k norms, written under `language_model.*` so the packer takes its qwen3_5 path (renames, the conv1d transpose, F32 `ssm_a` / `ssm_dt.bias`). The same rule, scoped to "not a conv layer", still refused every qwen3_5 hybrid: a GDN @@ -77,9 +84,10 @@ def write_safetensors(path, arrays): def make_model(d, np, family="qwen3"): rng = np.random.default_rng(1946) lfm2 = family == "lfm2" + lfm3 = family == "lfm3" q35 = family == "qwen3_5" - layers = 3 if lfm2 else LAYERS - conv_layers = (0, 2) if lfm2 else () + layers = 3 if lfm2 else (4 if lfm3 else LAYERS) + conv_layers = (0, 2) if (lfm2 or lfm3) else () gdn_layers = (0,) if q35 else () # a qwen3_5 pack nests its language model under language_model.* and ties the head to the embedding top = "language_model." if q35 else "model." @@ -93,7 +101,8 @@ def w(*shape, sc=0.05): def norm(n): return (1.0 + 0.1 * rng.standard_normal(n)).astype(np.float32) - weights = {top + "embed_tokens.weight": w(VOCAB, H, sc=0.5), top + "norm.weight": norm(H)} + weights = {top + "embed_tokens.weight": w(VOCAB, H, sc=0.5), + top + ("embedding_norm.weight" if lfm3 else "norm.weight"): norm(H)} if not q35: weights["lm_head.weight"] = w(VOCAB, H, sc=0.2) for layer in range(layers): @@ -113,6 +122,24 @@ def norm(n): p + "mlp.gate_proj.weight": w(FFN, H), p + "mlp.up_proj.weight": w(FFN, H), p + "mlp.down_proj.weight": w(H, FFN)}) continue + if lfm3: + # the upstream names, which model_load renames; the gate is 2 x 256, not a tile, so it stays bf16 + weights.update({p + "operator_norm.weight": norm(H), p + "post_operator_norm.weight": norm(H), + p + "ffn_norm.weight": norm(H), p + "post_ffn_norm.weight": norm(H), + p + "feed_forward.w1.weight": w(FFN, H), p + "feed_forward.w3.weight": w(FFN, H), + p + "feed_forward.w2.weight": w(H, FFN)}) + if layer in conv_layers: + weights.update({p + "conv.in_proj.weight": w(3 * H, H), p + "conv.conv.weight": w(H, 3, sc=0.4), + p + "conv.out_proj.weight": w(H, H)}) + else: + weights.update({ + p + "self_attn.q_proj.weight": w(HEADS * HEAD_DIM, H), + p + "self_attn.k_proj.weight": w(KV_HEADS * HEAD_DIM, H), + p + "self_attn.v_proj.weight": w(KV_HEADS * HEAD_DIM, H), + p + "self_attn.o_proj.weight": w(H, HEADS * HEAD_DIM), + p + "self_attn.q_norm.weight": norm(HEAD_DIM), p + "self_attn.k_norm.weight": norm(HEAD_DIM), + p + "self_attn.gate_proj.weight": w(HEADS, H, sc=0.3)}) + continue if layer in conv_layers: # in_proj is H -> 3H ([B | C | X]), the taps are [H][L] and stay bf16 (3 columns are not a tile), out_proj # is H -> H. No self_attn.* at all: that absence is what makes it a conv layer to the loader. @@ -148,6 +175,15 @@ def norm(n): del cfg["rms_norm_eps"] cfg.update({"architectures": ["Lfm2ForCausalLM"], "model_type": "lfm2", "num_hidden_layers": layers, "norm_eps": 1e-5, "conv_L_cache": 3, "full_attn_idxs": [1]}) + if lfm3: + del cfg["rms_norm_eps"], cfg["rope_theta"] + cfg.update({"architectures": ["Lfm3ForCausalLM"], "model_type": "lfm3", "num_hidden_layers": layers, + "norm_eps": 1e-5, "conv_L_cache": 3, "use_sandwich_norm": True, "sliding_window": 2, + "layer_types": ["conv", "full_attention", "conv", "sliding_attention"], + "rope_parameters": { + "full_attention": {"partial_rotary_factor": 0.5, "rope_theta": 1000000.0, + "rope_type": "default"}, + "sliding_attention": {"rope_theta": 10000, "rope_type": "default"}}}) with open(os.path.join(d, "config.json"), "w") as f: json.dump(cfg, f) with open(os.path.join(d, "tokenizer.json"), "w") as f: @@ -174,7 +210,7 @@ def tmp_root(): def main(): - if len(sys.argv) not in (2, 3) or (len(sys.argv) == 3 and sys.argv[2] not in ("lfm2", "qwen3_5")): + if len(sys.argv) not in (2, 3) or (len(sys.argv) == 3 and sys.argv[2] not in ("lfm2", "lfm3", "qwen3_5")): print(__doc__.strip().splitlines()[2]) return 2 family = sys.argv[2] if len(sys.argv) == 3 else "qwen3" diff --git a/tests/q4nx_lfm3_semantics_test.cpp b/tests/q4nx_lfm3_semantics_test.cpp new file mode 100644 index 000000000..1627d4c3f --- /dev/null +++ b/tests/q4nx_lfm3_semantics_test.cpp @@ -0,0 +1,161 @@ +/* Copyright 2026 bong-water-water-bong + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* q4nx_lfm3_semantics_test -- what the forward reads from an LFM3 config and container (engine#1786). + * + * FastFlowLM's LFM3-1.7B-NPU2 is LFM2's conv/attention hybrid with four more rules (LiquidAI's modeling_lfm3.py): + * layer_types names the conv layers ("conv") beside full and sliding attention; the norms are a sandwich around + * both blocks (use_sandwich_norm); each attention head's context is gated by sigmoid(gate_proj . x); and the full + * layers' partial RoPE keeps the full head_dim's frequency progression with a zero tail. Its container also keeps + * the upstream tensor names, which model_load renames to the canonical ones. Before this, the forward refused the + * pack by name at "conv". + * + * The config string is cut down from the pack's config.json to the fields the rules read. No model, no device. */ +#include "model.h" +#include "q4nx_semantics.h" + +#include +#include + +#include +#include +#include +#include +#include + +static int failures = 0; + +static void check(bool ok, const char* what) { + std::printf("%s %s\n", ok ? "ok " : "FAIL", what); + if (!ok) failures++; +} + +static int sem_of(const char* dir, const char* json, Q4nxSemantics* s) { + char path[512]; + std::snprintf(path, sizeof path, "%s/config.json", dir); + const int fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0600); + FILE* f = fd >= 0 ? fdopen(fd, "w") : nullptr; + if (!f) { std::printf("FAIL cannot write %s\n", path); failures++; return -1; } + std::fputs(json, f); + std::fclose(f); + if (q4nx_semantics_from_config(path, dir, s) != 0) { std::printf("FAIL cannot read %s\n", path); failures++; return -1; } + return 0; +} + +static const char* kLfm3 = + "{\"architectures\": [\"Lfm3ForCausalLM\"], \"model_type\": \"lfm3\", \"head_dim\": 128, \"hidden_size\": 2048," + " \"num_attention_heads\": 16, \"num_key_value_heads\": 4, \"num_hidden_layers\": 6, \"norm_eps\": 1e-05," + " \"layer_types\": [\"conv\", \"conv\", \"full_attention\", \"conv\", \"sliding_attention\", \"conv\"]," + " \"rope_parameters\": {\"full_attention\": {\"partial_rotary_factor\": 0.5, \"rope_theta\": 1000000.0," + " \"rope_type\": \"default\"}, \"sliding_attention\": {\"rope_theta\": 10000, \"rope_type\": \"default\"}}," + " \"sliding_window\": 256, \"use_sandwich_norm\": true, \"conv_L_cache\": 7}"; + +static int find(const std::vector& t, const char* name) { + for (size_t i = 0; i < t.size(); i++) + if (std::strcmp(t[i].name, name) == 0) return (int)i; + return -1; +} + +static std::vector table(const std::vector& names) { + std::vector t(names.size()); + for (size_t i = 0; i < names.size(); i++) { + std::memset(&t[i], 0, sizeof(TensorDesc)); + std::snprintf(t[i].name, sizeof(t[i].name), "%s", names[i]); + t[i].data_offset = (uint64_t)i; // the tag that follows a tensor through a rename + } + return t; +} + +int main() { + char dir[] = "/tmp/q4nx-lfm3-sem-XXXXXX"; + if (!mkdtemp(dir)) { std::printf("FAIL mkdtemp\n"); return 1; } + Q4nxSemantics s; + + if (sem_of(dir, kLfm3, &s) == 0) { + int conv = 0, full = 0, sliding = 0; + for (int i = 0; i < s.n_layer_types; i++) { + conv += s.layer_types[i] == Q4NX_LAYER_CONV; + full += s.layer_types[i] == Q4NX_LAYER_FULL; + sliding += s.layer_types[i] == Q4NX_LAYER_SLIDING; + } + check(s.unknown_layer_type[0] == '\0', "LFM3: \"conv\" in layer_types is a layer kind the forward knows"); + check(s.n_layer_types == 6 && conv == 4 && full == 1 && sliding == 1 && + s.layer_types[0] == Q4NX_LAYER_CONV && s.layer_types[2] == Q4NX_LAYER_FULL, + "... 4 conv, 1 full and 1 sliding layer, in the stated order"); + check(s.extra_layer_norms == 2, "use_sandwich_norm: the two extra norms around both blocks"); + check(s.attn_head_gate == 1, "LFM3 gates each attention head"); + check(s.proportional_rope == 1 && s.rotary_dim == 64 && s.rope_theta == 1000000.0f, + "full layers: zero-tail partial RoPE over the whole head (64 of 128 channels' frequencies), theta 1e6"); + check(s.rope_theta_sliding == 10000.0f, "sliding layers: their own theta"); + check(s.qk_norm == 1 && s.rmsnorm_offset == 0, "q/k norms on, plain norm form"); + } + if (sem_of(dir, "{\"model_type\": \"lfm3\", \"num_hidden_layers\": 2, \"use_sandwich_norm\": false," + " \"layer_types\": [\"conv\", \"full_attention\"]}", &s) == 0) + check(s.extra_layer_norms == 0, "use_sandwich_norm false: no extra norms"); + if (sem_of(dir, "{\"model_type\": \"lfm2\", \"hidden_size\": 1024, \"full_attn_idxs\": [2]}", &s) == 0) + check(s.attn_head_gate == 0 && s.extra_layer_norms == 0 && s.proportional_rope == 0, + "LFM2 keeps what it had: no gate, no sandwich, no zero-tail RoPE"); + if (sem_of(dir, "{\"model_type\": \"qwen3\", \"num_hidden_layers\": 2," + " \"layer_types\": [\"conv_attention\", \"full_attention\"]}", &s) == 0) + check(std::strcmp(s.unknown_layer_type, "conv_attention") == 0, + "an exact match: \"conv_attention\" is still an unknown kind, refused by name"); + + { + std::vector t = table({ + "model.embed_tokens.weight", "model.embedding_norm.weight", + "model.layers.0.conv.in_proj.weight", "model.layers.0.conv.conv.weight", "model.layers.0.conv.out_proj.weight", + "model.layers.0.operator_norm.weight", "model.layers.0.post_operator_norm.weight", + "model.layers.0.ffn_norm.weight", "model.layers.0.post_ffn_norm.weight", + "model.layers.0.feed_forward.w1.weight", "model.layers.0.feed_forward.w2.weight", + "model.layers.0.feed_forward.w3.weight", + "model.layers.1.operator_norm.weight", "model.layers.1.ffn_norm.weight", + "model.layers.1.self_attn.gate_proj.weight"}); + const int n = model_alias_lfm_names(t.data(), (int)t.size()); + auto at = [&](const char* name) { const int i = find(t, name); return i < 0 ? -1 : (int)t[(size_t)i].data_offset; }; + check(n == 13, "13 of the 15 names are renamed (the embedding and the gate keep theirs)"); + check(at("model.norm.weight") == 1, "embedding_norm is the final norm"); + check(at("model.layers.0.shortconv.in_proj.weight") == 2 && at("model.layers.0.shortconv.conv.weight") == 3 && + at("model.layers.0.shortconv.out_proj.weight") == 4, + "conv.* -> shortconv.* (conv.conv is the taps)"); + check(at("model.layers.0.input_layernorm.weight") == 5, "operator_norm -> input_layernorm"); + check(at("model.layers.0.post_attention_layernorm.weight") == 6 && + at("model.layers.0.pre_feedforward_layernorm.weight") == 7 && + at("model.layers.0.post_feedforward_layernorm.weight") == 8, + "sandwich layer: post_operator_norm is the block output's norm, ffn_norm the MLP input's"); + check(at("model.layers.0.mlp.gate_proj.weight") == 9 && at("model.layers.0.mlp.down_proj.weight") == 10 && + at("model.layers.0.mlp.up_proj.weight") == 11, + "w1 is gate, w2 is down, w3 is up (Lfm3MLP: w2(silu(w1 x) * w3 x))"); + check(at("model.layers.1.post_attention_layernorm.weight") == 13, + "a layer without post_operator_norm (LFM2's upstream form): ffn_norm is post_attention_layernorm"); + check(at("model.layers.1.self_attn.gate_proj.weight") == 14, "the attention gate keeps its name"); + } + { + std::vector t = table({"model.norm.weight", "model.embedding_norm.weight", + "model.layers.0.input_layernorm.weight", "model.layers.0.operator_norm.weight"}); + check(model_alias_lfm_names(t.data(), (int)t.size()) == 0 && find(t, "model.embedding_norm.weight") == 1 && + find(t, "model.layers.0.operator_norm.weight") == 3, + "a canonical name already present is never overwritten"); + std::vector q = table({"model.layers.0.shortconv.conv.weight", "model.layers.0.mlp.gate_proj.weight"}); + check(model_alias_lfm_names(q.data(), (int)q.size()) == 0, "an LFM2 pack's canonical names are left alone"); + } + + char path[512]; + std::snprintf(path, sizeof path, "%s/config.json", dir); + unlink(path); + rmdir(dir); + std::printf(failures ? "q4nx_lfm3_semantics: %d FAILED\n" : "q4nx_lfm3_semantics: ok\n", failures); + return failures ? 1 : 0; +} From 583051ba5f6cdebaf15c6a1acb2c43c761cc5401 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 12 Nov 2026 01:43:49 -0400 Subject: [PATCH 2/3] tests: the lfm3 arm checks the forward against a modeling_lfm3 reference, not only against itself (Refs #1786) Review of #2254 at 03a3066: batched == sequential compares our forward with itself, so the head gate made a no-op in both paths still passed. batch_check_state_test now writes its sequential logits when ONEBIT_TEST_LOGITS_OUT is set, and the lfm3 arm runs LFM3's math in numpy on the container's own weights (I8 untiled with q4nx_pack.untile) and requires corr >= 0.99999 and max |d| <= 1e-3 of the largest logit. Measured: corr 1.0000000, max |d| 1.2e-05 of 7.63. Mutants that pass batched == sequential and fail this: gate dead in both paths (corr 0.898, argmax 137 vs 234), no zero-tail RoPE (0.976, 50), no sandwich (0.377, 83). Co-Authored-By: Claude Opus 5.5 --- tests/batch_check_state_ci.py | 100 ++++++++++++++++++++++++++++++- tests/batch_check_state_test.cpp | 11 ++++ 2 files changed, 110 insertions(+), 1 deletion(-) diff --git a/tests/batch_check_state_ci.py b/tests/batch_check_state_ci.py index 0a3bdbaef..b52a96e59 100755 --- a/tests/batch_check_state_ci.py +++ b/tests/batch_check_state_ci.py @@ -39,6 +39,10 @@ the zero-tail partial RoPE on the full layer, the sandwich norms, and a bf16 per-head output gate (self_attn.gate_proj [heads, hidden]). What it pins: model_load's renaming, the forward accepting "conv" in layer_types, and the batched prefill running the gate and the sandwich exactly as the token-by-token path does. +batched == sequential compares the forward with itself, so this arm also runs LFM3's math in numpy (lfm3_reference, +transcribed from modeling_lfm3.py) on the container's own weights, untiled, and compares the sequential logits: +measured at engine#2254, corr 1.0000000 and max |d| 1.2e-05 of 7.63. A dead head gate (in both paths) gives corr 0.90 +and another argmax, the usual partial RoPE 0.98, no sandwich 0.38; every one of them passes batched == sequential. With `qwen3_5` it is a qwen3_5 hybrid (engine#1889): a linear-attention (GDN) layer, then a full-attention layer with q/k norms, written under `language_model.*` so the packer takes its qwen3_5 path (renames, the conv1d transpose, @@ -62,6 +66,8 @@ TOOL = os.path.join(HERE, "..", "tools", "safetensors_to_q4nx.py") H, HEADS, KV_HEADS, HEAD_DIM, FFN, VOCAB, LAYERS = 256, 2, 1, 128, 512, 256, 2 +LFM3_MIN_CORR = 0.99999 # measured at #2254: corr 1.0000000, max |d| 1.2e-05 of 7.63 +LFM3_MAX_DIFF = 1e-3 # of the reference's largest |logit| def write_safetensors(path, arrays): @@ -190,6 +196,84 @@ def norm(n): f.write("{}") with open(os.path.join(d, "tokenizer_config.json"), "w") as f: json.dump({"add_bos_token": False}, f) + return {name: a.shape for name, a in weights.items()} + + +def lfm3_reference(q4nx_dir, np, ids, shapes): + """LFM3's forward (LiquidAI's modeling_lfm3.py, transcribed) on the weights the forward itself reads: the packed + container's tensors, the I8 ones untiled. Returns the last position's logits. + + batched == sequential compares our forward with itself, so a rule both paths get wrong passes it (#2254 review: + with the head gate a no-op in step() and step_batch() alike, the arm still matched). This is the check that + can see the gate, the zero-tail RoPE, the sandwich and the conv layers.""" + sys.path.insert(0, os.path.join(HERE, "..", "tools")) + from q4nx_pack import untile + with open(os.path.join(q4nx_dir, "config.json")) as f: + cfg = json.load(f) + with open(os.path.join(q4nx_dir, "model.q4nx"), "rb") as f: + n = struct.unpack("hts", q, k) / np.sqrt(hd) + mask = np.tril(np.ones((T, T), bool)) + if lt == "sliding_attention": + mask &= (np.arange(T)[:, None] - np.arange(T)[None, :]) < win + sc = np.exp(np.where(mask[None], sc, -np.inf) - sc.max(-1, keepdims=True)) + ctx = np.einsum("hts,shd->thd", sc / sc.sum(-1, keepdims=True), v) + gate = 1.0 / (1.0 + np.exp(-(xn @ W(p + "self_attn.gate_proj.weight").T))) + y = (ctx * gate[:, :, None]).reshape(T, nh * hd) @ W(p + "self_attn.o_proj.weight").T + x = x + rms(y, W(p + "post_operator_norm.weight")) + h = rms(x, W(p + "ffn_norm.weight")) + a = h @ W(p + "feed_forward.w1.weight").T + ff = ((a / (1.0 + np.exp(-a))) * (h @ W(p + "feed_forward.w3.weight").T)) @ W(p + "feed_forward.w2.weight").T + x = x + rms(ff, W(p + "post_ffn_norm.weight")) + return rms(x, W("model.embedding_norm.weight"))[-1] @ W("lm_head.weight").T def tmp_root(): @@ -221,12 +305,15 @@ def main(): return 77 with tempfile.TemporaryDirectory(prefix="batch_check_state_ci.", dir=tmp_root()) as tmp: src, out = os.path.join(tmp, "hf"), os.path.join(tmp, "q4nx") - make_model(src, np, family) + shapes = make_model(src, np, family) r = subprocess.run([sys.executable, TOOL, "pack", src, out], capture_output=True, text=True) if r.returncode != 0: print(f"FAIL packing the fabricated container: {r.stderr[-600:]}") return 1 env = dict(os.environ, ONEBIT_TEST_Q4NX=out) + logits_out = os.path.join(tmp, "sequential.f32") + if family == "lfm3": + env["ONEBIT_TEST_LOGITS_OUT"] = logits_out env.pop("NPU_INFER_HOST_GEMM", None) # init_host is host-GEMM already; nothing else should steer it r = subprocess.run([sys.argv[1]], env=env, capture_output=True, text=True, timeout=110) for line in (r.stdout + r.stderr).splitlines(): @@ -238,6 +325,17 @@ def main(): # arm, not a reason to skip it. print("FAIL batch_check_state_test skipped on the fabricated container (init_host refused it)") return 1 + if family == "lfm3" and r.returncode == 0: + ours = np.fromfile(logits_out, dtype=np.float32)[:VOCAB] + ref = lfm3_reference(out, np, [1, 2, 3, 4, 5, 6], shapes) + corr = float(np.corrcoef(ours, ref)[0, 1]) + ok = (corr >= LFM3_MIN_CORR and float(np.abs(ours - ref).max()) <= LFM3_MAX_DIFF * float(np.abs(ref).max()) + and int(np.argmax(ours)) == int(np.argmax(ref))) + print(f"{'PASS' if ok else 'FAIL'} sequential logits vs the modeling_lfm3 reference on the container's " + f"weights: corr {corr:.7f} (needs {LFM3_MIN_CORR}), max |d| {float(np.abs(ours - ref).max()):.3g} of " + f"{float(np.abs(ref).max()):.3g}, argmax {int(np.argmax(ours))} vs " + f"{int(np.argmax(ref))}") + return 0 if ok else 1 return r.returncode diff --git a/tests/batch_check_state_test.cpp b/tests/batch_check_state_test.cpp index 74fdf5308..4533e1d3f 100644 --- a/tests/batch_check_state_test.cpp +++ b/tests/batch_check_state_test.cpp @@ -104,6 +104,17 @@ int main() { return 1; } } + // A caller that has its own reference forward of the container (tests/batch_check_state_ci.py lfm3) asks for the + // sequential logits: batched == sequential cannot see a rule both paths get wrong, a reference can. + if (const char* out = std::getenv("ONEBIT_TEST_LOGITS_OUT")) { + FILE* f = std::fopen(out, "wb"); + if (!f || std::fwrite(ref.data(), sizeof(float), ref.size(), f) != ref.size()) { + std::printf("FAIL cannot write the sequential logits to %s\n", out); + if (f) std::fclose(f); + return 1; + } + std::fclose(f); + } fwd.reset(); std::vector logits; From ca597d4a9f067cb1ee528e9ed5aa54ddf4d7cfec Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 12 Nov 2026 02:01:02 -0400 Subject: [PATCH 3/3] tests: the sequential-logits dump is created 0600 and never through a symlink (Refs #1786) Review of #2254 at 583051b (CodeQL cpp/world-writable-file-creation, introduced by this PR): fopen(out, "wb") created the ONEBIT_TEST_LOGITS_OUT file 0666 & ~umask. It now goes through file_create.h's onebit_fopen_private_wb. Measured under umask 0002: the dump is 0600; a symlink at the path is refused (rc 1, "cannot write", no target file created). The lfm3 arm passes as before (corr 1.0000000, max |d| 1.19e-05). Co-Authored-By: Claude Opus 5.5 --- tests/batch_check_state_test.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/batch_check_state_test.cpp b/tests/batch_check_state_test.cpp index 4533e1d3f..8150c7f3f 100644 --- a/tests/batch_check_state_test.cpp +++ b/tests/batch_check_state_test.cpp @@ -36,6 +36,7 @@ // Where it runs: in CI (engine#1946), built against the public runtime stub and run by tests/batch_check_state_ci.py // on a two-layer container that script fabricates; and on a dev box with ONEBIT_NPU=ON, on the container named by // ONEBIT_TEST_Q4NX. +#include "file_create.h" #include "q4nx_forward.h" #include @@ -107,7 +108,7 @@ int main() { // A caller that has its own reference forward of the container (tests/batch_check_state_ci.py lfm3) asks for the // sequential logits: batched == sequential cannot see a rule both paths get wrong, a reference can. if (const char* out = std::getenv("ONEBIT_TEST_LOGITS_OUT")) { - FILE* f = std::fopen(out, "wb"); + FILE* f = onebit_fopen_private_wb(out); // 0600, never through a symlink (file_create.h) if (!f || std::fwrite(ref.data(), sizeof(float), ref.size(), f) != ref.size()) { std::printf("FAIL cannot write the sequential logits to %s\n", out); if (f) std::fclose(f);