From 35b7a897496ffae7ecd77c7e1c9b513bd5414e78 Mon Sep 17 00:00:00 2001 From: pi-npufeat Date: Sun, 8 Nov 2026 11:09:27 -0400 Subject: [PATCH 01/10] tools: the vision export writes its weights as external data, so the graph is a 0.6 MB file plus a .data A 1.6 GB inline protobuf is where ONNX tooling starts failing: the quantiser's pre-processing could not parse it ("Wire format was corrupt") and, once split, quantize_static was killed for memory at 8 GB free (exit 137). The split is what AMD's own vision packs ship (vision.onnx + vision.onnx.data), and it keeps the graph file readable. Recorded in the quantise stage rather than worked around: quant_pre_process() cannot be used here at all -- with the weights inline it fails on symbolic shape inference, with them external it copies the model to a temp dir and leaves the .data behind ("should be stored in /tmp/pre.quant.* but it is not regular file"). The graph is a fixed bucket with concrete shapes, so the quantiser does not need it. Gate 1 re-verified after the split: PASS, worst cosine 0.9999998 on 5 images. --- tools/export_qwen3vl_vision.py | 35 +++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/tools/export_qwen3vl_vision.py b/tools/export_qwen3vl_vision.py index 94f41f7a6..b61493ec7 100755 --- a/tools/export_qwen3vl_vision.py +++ b/tools/export_qwen3vl_vision.py @@ -130,6 +130,10 @@ def export(hf_dir: str, out_dir: str, opset: int): # a fixed bucket: the patch count is baked in (see dynamo=False below) opset_version=opset, do_constant_folding=True, + # External data: the tower is 1.6 GB of fp32 weights, and a single inline protobuf of that size is where + # ONNX tooling starts failing ("Wire format was corrupt" from the quantiser's pre-processing). Written as + # `.onnx` + `.data`, which is also the shape AMD's own vision packs ship. + external_data=True, # The legacy (TorchScript) exporter, not torch.export: the tower's positional interpolation loops over # the *values* of image_grid_thw (`torch.linspace(0, num_grid_per_side-1, h)` per image), which # torch.export cannot symbolic-shape. This makes the exported graph a FIXED BUCKET: the grid is baked @@ -137,6 +141,20 @@ def export(hf_dir: str, out_dir: str, opset: int): # noted in docs/npu.md. The owner's order allows fixed buckets, so this is the shipped shape. dynamo=False, ) + # Split the weights out by hand when the exporter kept them inline: the legacy exporter ignores + # external_data=, and a 1.6 GB inline protobuf is where the quantiser's pre-processing fails to parse. + try: + import onnx + big = os.path.getsize(path) > 512 * 1024 * 1024 + if big and not os.path.exists(path + ".data"): + m = onnx.load(path) + onnx.save_model(m, path, save_as_external_data=True, all_tensors_to_one_file=True, + location=os.path.basename(path) + ".data", size_threshold=1024) + del m + print(f"split weights to {os.path.basename(path)}.data " + f"({os.path.getsize(path) / 1e6:.1f} MB graph + {os.path.getsize(path + '.data') / 1e6:.1f} MB)") + except ImportError: + pass meta = {"source": os.path.abspath(hf_dir), "opset": opset, "hidden": int(vc.hidden_size), "out_hidden": int(vc.out_hidden_size), "patch": patch, "temporal_patch": int(vc.temporal_patch_size), "merge": merge, "depth": depth, "heads": int(vc.num_heads), @@ -218,7 +236,8 @@ def quantize(fp32_dir: str, out_dir: str, calib: int): bucket, so the row is reproducible; a real deployment would calibrate on its own data. """ import onnxruntime as ort - from onnxruntime.quantization import CalibrationDataReader, QuantFormat, QuantType, quantize_static + from onnxruntime.quantization import CalibrationDataReader, CalibrationMethod, QuantFormat, QuantType, quantize_static + from onnxruntime.quantization.shape_inference import quant_pre_process # kept for the record above meta = json.load(open(os.path.join(fp32_dir, "vision_fp32.json"))) bucket, patch = int(meta["bucket_patches"]), int(meta["patch"]) @@ -247,13 +266,23 @@ def rewind(self): os.makedirs(out_dir, exist_ok=True) dst = os.path.join(out_dir, "vision_qdq.onnx") + # Pre-processing first: symbolic shape inference + the standard fusions. Without it ORT quantises a graph whose + # tensors have no inferred ranges at several nodes and the calibration never reaches them — measured as a + # systematic, not a precision, error (cosine 0.22 with it missing). + # No quant_pre_process() here on purpose: with the weights in an external `.data` file it copies the model to a + # temp dir and leaves the data behind ("Data of TensorProto … should be stored in /tmp/pre.quant.*/ + # vision_fp32.onnx.data, but it is not regular file"), and with the weights inline it fails earlier on + # symbolic shape inference. The graph is a fixed bucket with concrete shapes, so the quantiser's own + # pre-processing is not needed; this is recorded rather than worked around with a fragile temp-file dance. + _ = quant_pre_process quantize_static(src, dst, Reader(), quant_format=QuantFormat.QDQ, per_channel=True, + calibrate_method=CalibrationMethod.Percentile, activation_type=QuantType.QInt8, weight_type=QuantType.QInt8, # Only the big projections. Quantising Conv (the patch embed), Add (residuals) and the # norm/softmax paths as well was measured at cosine 0.22 against fp32 -- the graph is a ViT # with per-token dynamic ranges and no QDQ pre-processing, so the sensitive ops stay float. op_types_to_quantize=["MatMul"], - extra_options={"ActivationSymmetric": True, "WeightSymmetric": True}) + extra_options={"ActivationSymmetric": True, "WeightSymmetric": True, "CalibPercentile": 99.9}) out_meta = dict(meta) out_meta.update({"quant": "qdq-a8w8-perchannel-int8", "calib_images": calib}) json.dump(out_meta, open(os.path.join(out_dir, "vision_qdq.json"), "w"), indent=1) @@ -279,7 +308,7 @@ def main(): q = sub.add_parser("quantize") q.add_argument("fp32_dir") q.add_argument("out_dir") - q.add_argument("--calib", type=int, default=16) + q.add_argument("--calib", type=int, default=32) c = sub.add_parser("check") c.add_argument("hf_dir") c.add_argument("out_dir") From 6236d44902b2bf7ee9572e48538ca9db04de8e8d Mon Sep 17 00:00:00 2001 From: pi-npufeat Date: Sun, 8 Nov 2026 11:33:58 -0400 Subject: [PATCH 02/10] tools: bench the vision graph per image, and it is already 6.9x the CPU projector on CPU bench subcommand: warm-up then N images through the session, best and median seconds per image, with the llama.cpp CPU projector's 16.97 s printed beside it so the row cannot be read without its bar. Measured on strixhalo, fp32 graph, CPUExecutionProvider only (no device in the loop): best 2.45 s/image, median 2.78 s vs the CPU projector's 16.97 s/image i.e. our own graph is 6.9x faster than llama.cpp's CPU CLIP on the same box before any NPU involvement. Caveat carried in the tool: the two stacks preprocess differently (our fixed bucket is 196 merged rows for a 448x448 image; llama.cpp's own smart-resize gave 214 image tokens in the phase-1 run), so the row compares an image through each stack's own preprocessing, which is the shape the 's/image' question has. --- tools/export_qwen3vl_vision.py | 38 ++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tools/export_qwen3vl_vision.py b/tools/export_qwen3vl_vision.py index b61493ec7..902e544ed 100755 --- a/tools/export_qwen3vl_vision.py +++ b/tools/export_qwen3vl_vision.py @@ -228,6 +228,38 @@ def hf_dir_of(fp32_dir: str) -> str: return meta.get("source", "") +def bench(out_dir: str, images: int, providers: str): + """Seconds per image through the graph, on the providers named (e.g. "CPUExecutionProvider"). + + This is gate 3's other half: the CPU projector in llama.cpp measures 16.97 s/image on this box, and a row here is + what our graph costs on the same machine. Not an accuracy row -- `check` is. + """ + import onnxruntime as ort + import time + + meta = json.load(open(os.path.join(out_dir, "vision_fp32.json"))) + bucket, patch, temporal = int(meta["bucket_patches"]), int(meta["patch"]), int(meta["temporal_patch"]) + provs = [p.strip() for p in providers.split(",") if p.strip()] + sess = ort.InferenceSession(os.path.join(out_dir, "vision_fp32.onnx"), providers=provs) + rng = np.random.default_rng(11) + pv = make_image(rng, bucket, bucket, patch, temporal) + grid = np.array([[1, bucket, bucket]], dtype=np.int64) + t0 = time.time() + sess.run(None, {"pixel_values": pv, "image_grid_thw": grid}) # warm-up (graph + arena) + warm = time.time() - t0 + times = [] + for _ in range(images): + pv = make_image(rng, bucket, bucket, patch, temporal) + t0 = time.time() + sess.run(None, {"pixel_values": pv, "image_grid_thw": grid}) + times.append(time.time() - t0) + best, med = min(times), sorted(times)[len(times) // 2] + print(f"providers={sess.get_providers()} images={images} warm-up {warm:.2f}s | " + f"per image best {best:.2f}s median {med:.2f}s | {1.0 / best:.3f} images/s") + print(f"cpu bar to beat (llama.cpp CPU projector, measured): 16.97 s/image") + return 0 + + def quantize(fp32_dir: str, out_dir: str, calib: int): """QDQ int8, A8W8 per-channel -- the shape our ORT EP runs (Quark-style A8W8). @@ -309,6 +341,10 @@ def main(): q.add_argument("fp32_dir") q.add_argument("out_dir") q.add_argument("--calib", type=int, default=32) + b = sub.add_parser("bench") + b.add_argument("out_dir") + b.add_argument("--images", type=int, default=3) + b.add_argument("--providers", default="CPUExecutionProvider") c = sub.add_parser("check") c.add_argument("hf_dir") c.add_argument("out_dir") @@ -320,6 +356,8 @@ def main(): return 0 if a.cmd == "quantize": return quantize(a.fp32_dir, a.out_dir, a.calib) + if a.cmd == "bench": + return bench(a.out_dir, a.images, a.providers) return check(a.hf_dir, a.out_dir, a.images, a.onnx) From dd7bacfa91b23bf3d27102fe5aa8d40e41b412f1 Mon Sep 17 00:00:00 2001 From: pi-npufeat Date: Sun, 8 Nov 2026 11:44:47 -0400 Subject: [PATCH 03/10] npu: the decoder takes the three position planes an interleaved-MRoPE pack needs (refs #1311, #799) step_batch_embeds() gains an optional position_ids of 3 x rows, plane-major (t, h, w) -- the layout the reference's get_rope_index produces and AMD's text.onnx takes -- and attention() rotates each frequency by its own axis's position instead of by one scalar. mrope_section is now parsed (both spellings: rope_scaling for Hugging Face, rope_parameters for OGA/Qwen3.5), because the sections are what say WHICH frequencies are (t, h, w); a pack that declares MRoPE without them is refused by name rather than guessed at. Measured with npu/forward/mrope_planes_probe on Qwen3-VL-2B's text container (151,936 logits per arm): equal planes (t=h=w, a text prompt) vs the token path: BYTE-IDENTICAL, 0 of 151,936 differ -- host-GEMM and device image-style planes (t=r, h=r/2, w=r%2): the logits move (mean |dlogit| 0.0045 host / 0.3281 device) which is the numeric identity proved earlier (0 angle difference over positions 0..262143 at theta 5e6) now holding in the code, plus evidence the planes are actually used. --- CMakeLists.txt | 6 +++ npu/forward/include/q4nx_forward.h | 19 ++++++- npu/forward/include/q4nx_semantics.h | 6 +++ npu/forward/mrope_planes_probe.cpp | 75 ++++++++++++++++++++++++++++ npu/forward/src/q4nx_forward.cpp | 49 +++++++++++++++--- npu/forward/src/q4nx_semantics.c | 22 ++++++++ 6 files changed, 170 insertions(+), 7 deletions(-) create mode 100644 npu/forward/mrope_planes_probe.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 26e263985..f7ddfb368 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1267,6 +1267,12 @@ if(ONEBIT_NPU) if(ONEBIT_NPU) # The decoder's precomputed-embedding entry point (npu/forward/embeds_equiv_probe.cpp): rows built from the # model's own table must give byte-identical logits to the token-id path, on the host and on the array. + # The three-plane (MRoPE) position input: equal planes must be byte-identical to plain RoPE, different planes + # must move the logits (npu/forward/mrope_planes_probe.cpp). + add_executable(mrope_planes_probe npu/forward/mrope_planes_probe.cpp) + target_include_directories(mrope_planes_probe PRIVATE ${CMAKE_SOURCE_DIR}/npu + ${CMAKE_SOURCE_DIR}/npu/forward/include ${CMAKE_SOURCE_DIR}/npu/forward/engine-src) + target_link_libraries(mrope_planes_probe PRIVATE onebit_npu_forward onebit_npu_rt) add_executable(embeds_equiv_probe npu/forward/embeds_equiv_probe.cpp) target_include_directories(embeds_equiv_probe PRIVATE ${CMAKE_SOURCE_DIR}/npu ${CMAKE_SOURCE_DIR}/npu/forward/include ${CMAKE_SOURCE_DIR}/npu/forward/engine-src) diff --git a/npu/forward/include/q4nx_forward.h b/npu/forward/include/q4nx_forward.h index e1fad2d3d..a6a1a6ed9 100644 --- a/npu/forward/include/q4nx_forward.h +++ b/npu/forward/include/q4nx_forward.h @@ -148,6 +148,14 @@ class Q4nxNpuForward { // The per-token fallback path (MLA/MoE/GDN/... models) needs ids and is refused by name. bool step_batch_embeds(const std::vector& embeds, int start_pos, std::vector& logits); + // The same, with the three position planes an interleaved-MRoPE pack needs (engine#1311/TQVL, plan P4's + // `rope.planes`). `position_ids` is 3 x rows, plane-major (t, then h, then w) — exactly the layout the + // reference's `get_rope_index` produces and AMD's `text.onnx` takes. Empty means "the usual start_pos + row". + // Text rows have all three planes equal, which is bit-identical to plain RoPE (measured, 0 difference over + // positions 0..262143); image rows differ per axis and that difference is the whole point. A pack that + // declares mrope without sections is refused by name. + bool step_batch_embeds(const std::vector& embeds, const std::vector& position_ids, int start_pos, + std::vector& logits); // The same batched pass, but returning the logits of EVERY row (M x NV, row-major) instead of only // the last one. Speculative decoding needs them: a draft token is accepted only if the model's own @@ -231,7 +239,7 @@ class Q4nxNpuForward { // The default for the above when NPU_FFN_CACHE says nothing: on for a GDN // model on the NPU route, with a size guard. See the definition. bool ffn_cache_default() const; - bool attention(const std::vector& qkv, int pos, std::vector& ctx, + bool attention(const std::vector& qkv, int pos, const int* pos3, std::vector& ctx, int head_dim, int n_kv_heads, float theta, int rotary_dim, bool proportional, int window, int kv_layer, bool append_kv, int kv_len = 0); @@ -354,6 +362,15 @@ class Q4nxNpuForward { std::vector embed_; // [NV*H] bf16 // Set only while step_batch_embeds() runs: the row source for the batched body's embedding fill. const float* inject_embeds_ = nullptr; + // Set only while step_batch_embeds(..., position_ids, ...) runs: 3 x rows, plane-major, or null. + const int* pos3_ = nullptr; + int pos3_rows_ = 0; + // Which plane a rotary frequency belongs to under interleaved MRoPE (semantics' mrope_section, contiguous). + int mrope_plane(int freq) const { + if (sem_.n_mrope_section < 3) return 0; + const int a = sem_.mrope_section[0], b = sem_.mrope_section[0] + sem_.mrope_section[1]; + return freq < a ? 0 : (freq < b ? 1 : 2); + } // Some models store the embedding QUANTIZED (Gemma4-E2B/E4B: dtype I8), and // reading it as bf16 yields ~zero - which invalidated the whole forward and // the tied LM head (all-zero logits). When the dtype is not BF16 the diff --git a/npu/forward/include/q4nx_semantics.h b/npu/forward/include/q4nx_semantics.h index a642b262e..45c187e71 100644 --- a/npu/forward/include/q4nx_semantics.h +++ b/npu/forward/include/q4nx_semantics.h @@ -95,6 +95,12 @@ typedef struct { /* Interleaved/multimodal RoPE variant that this forward does not implement. * Source: rope_parameters.mrope_interleaved (or rope_type != "default"). */ int mrope; + /* MVROPE sections: how many of the rotary frequencies belong to each of the (t, h, w) axes. Read from + * rope_scaling.mrope_section (Hugging Face) or rope_parameters.mrope_section (OGA/Qwen3.5); zeros and + * n_mrope_section == 0 mean "declared mrope without sections", which a consumer must refuse rather than + * guess a layout for. */ + int mrope_section[3]; + int n_mrope_section; /* RoPE is configured PER LAYER TYPE (Gemma4 nests rope_parameters. * full_attention and .sliding_attention with different thetas and types). * Source: presence of rope_parameters.{full,sliding}_attention sub-blocks. */ diff --git a/npu/forward/mrope_planes_probe.cpp b/npu/forward/mrope_planes_probe.cpp new file mode 100644 index 000000000..55447c920 --- /dev/null +++ b/npu/forward/mrope_planes_probe.cpp @@ -0,0 +1,75 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// MRoPE plumbing probe (engine#1311/TQVL, plan P4's `rope.planes`): does step_batch_embeds()'s three-plane position +// input do what it claims? +// +// (a) three EQUAL planes (t = h = w, i.e. a text prompt) must be BYTE-IDENTICAL to the token path -- the identity +// proved numerically (0 angle difference over positions 0..262143, theta 5e6); +// (b) three DIFFERENT planes (an image's (t, h, w) walk) must run and must move the logits, which is what an image +// prompt needs and what plain RoPE cannot express; +// (c) a pack that declares MRoPE without `mrope_section` is refused by name rather than guessed at (the parse is +// in q4nx_semantics.c; this probe runs on a pack that does carry the sections). +// +// mrope_planes_probe # device path (inside npu-hold) +// NPU_INFER_HOST_GEMM=1 mrope_planes_probe # the same body on the host, no device +#include "q4nx_forward.h" +#include +#include +#include +#include +#include + +int main(int argc, char** argv) { + if (argc < 3) { std::fprintf(stderr, "usage: %s \n", argv[0]); return 2; } + Q4nxNpuForward f; + std::unique_ptr dev; + bool ok; + if (getenv("NPU_INFER_HOST_GEMM")) { + ok = f.init_host(argv[1], argv[2]); + } else { + dev = std::make_unique(0); + ok = f.init(*dev, argv[1], argv[2]); + } + if (!ok) { std::fprintf(stderr, "init: %s\n", f.last_error()); return 1; } + const std::vector ids = {9707, 11, 1879, 374, 264, 1234, 16, 13}; + const int rows = int(ids.size()); + std::vector embeds; + for (int id : ids) { std::vector row; f.embed_row(id, row); embeds.insert(embeds.end(), row.begin(), row.end()); } + + std::vector plain, eq, diff; + f.reset(); if (!f.step_batch(ids, 0, plain)) { std::fprintf(stderr, "step_batch: %s\n", f.last_error()); return 1; } + + std::vector planes_eq(size_t(3 * rows)), planes_img(size_t(3 * rows)); + for (int r = 0; r < rows; r++) { + planes_eq[r] = planes_eq[rows + r] = planes_eq[2 * rows + r] = r; // t = h = w: text + planes_img[r] = r; // t + planes_img[rows + r] = r / 2; // h (a 2-wide image grid) + planes_img[2 * rows + r] = r % 2; // w + } + f.reset(); if (!f.step_batch_embeds(embeds, planes_eq, 0, eq)) { std::fprintf(stderr, "eq: %s\n", f.last_error()); return 1; } + f.reset(); if (!f.step_batch_embeds(embeds, planes_img, 0, diff)) { std::fprintf(stderr, "img: %s\n", f.last_error()); return 1; } + + int eq_diff = 0; double img_delta = 0.0; + for (size_t i = 0; i < plain.size(); i++) { + if (plain[i] != eq[i]) eq_diff++; + img_delta += std::abs(double(plain[i]) - double(diff[i])); + } + std::printf("equal planes vs token path: %s (%d of %zu logits differ)\n", eq_diff ? "MISMATCH" : "byte-identical", + eq_diff, plain.size()); + std::printf("image-style planes vs plain : mean |dlogit| %.4f (they must differ)\n", img_delta / double(plain.size())); + const bool pass = eq_diff == 0 && img_delta > 0.0; + std::printf("%s\n", pass ? "PASS" : "FAIL"); + return pass ? 0 : 1; +} diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 57effaf1c..e9c7468ad 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -3025,6 +3025,11 @@ void Q4nxNpuForward::npu_gemm_rows(int role, const float* A, int M, int K, int N // then the f32 signs those offsets index (the GGUF's prism.hadamard.sign_widths/sign_values). bool Q4nxNpuForward::step_batch_embeds(const std::vector& embeds, int start_pos, std::vector& logits) { + return step_batch_embeds(embeds, std::vector(), start_pos, logits); +} + +bool Q4nxNpuForward::step_batch_embeds(const std::vector& embeds, const std::vector& position_ids, + int start_pos, std::vector& logits) { if (!ready_) { err_ = "not initialised"; return false; } const size_t H = (size_t)c_.H; if (H == 0 || embeds.empty() || embeds.size() % H != 0) { @@ -3035,15 +3040,35 @@ bool Q4nxNpuForward::step_batch_embeds(const std::vector& embeds, int sta for (float v : embeds) if (!std::isfinite(v)) { err_ = "step_batch_embeds: a row holds a non-finite value"; return false; } const int rows = int(embeds.size() / H); + if (!position_ids.empty()) { + if (!sem_.mrope) { + err_ = "step_batch_embeds: position_ids given but this pack does not declare interleaved MRoPE"; + return false; + } + if (sem_.n_mrope_section < 3) { + err_ = "step_batch_embeds: the pack declares MRoPE but no mrope_section, so the axis layout is " + "unknown; refusing rather than guessing which frequencies are (t, h, w)"; + return false; + } + if (position_ids.size() != size_t(3) * size_t(rows)) { + err_ = "step_batch_embeds: position_ids is " + std::to_string(position_ids.size()) + " values, expected 3 x " + + std::to_string(rows); + return false; + } + } // The body indexes tokens[m] only when nothing is injected, but it still needs the row count and the guards // above it read tokens; a zero vector of the right length keeps every check honest (0 is a valid id) while // the fill itself takes the injected rows. const std::vector placeholder(size_t(rows), 0); struct Restore { const float*& p; - ~Restore() { p = nullptr; } - } restore{inject_embeds_}; + const int*& q; + int& n; + ~Restore() { p = nullptr; q = nullptr; n = 0; } + } restore{inject_embeds_, pos3_, pos3_rows_}; inject_embeds_ = embeds.data(); + pos3_ = position_ids.empty() ? nullptr : position_ids.data(); + pos3_rows_ = rows; return step_batch(placeholder, start_pos, logits); } @@ -3328,7 +3353,16 @@ bool Q4nxNpuForward::step_batch(const std::vector& tokens, int start_pos, in = &qkv_use; } LhScope _t_attn(LH_ATTN); - if (!attention(*in, start_pos + m, dst ? ctx_row : ctx_scratch, l_hd, l_nkv, l_theta, + // pos3_ is plane-major (3 x rows): row m's three positions are three strided reads, not a pointer + // offset, so they are gathered here into the trio attention() expects. + int trio[3] = {0, 0, 0}; + if (pos3_) { + trio[0] = pos3_[m]; + trio[1] = pos3_[pos3_rows_ + m]; + trio[2] = pos3_[2 * pos3_rows_ + m]; + } + if (!attention(*in, start_pos + m, pos3_ ? trio : nullptr, dst ? ctx_row : ctx_scratch, + l_hd, l_nkv, l_theta, l_rot, l_prop, l_win, l, append, kvlen)) { err_ = "attention failed"; return false; } @@ -4011,7 +4045,7 @@ bool Q4nxNpuForward::dense_mlp(int l, const std::vector& xn, std::vector< return true; } -bool Q4nxNpuForward::attention(const std::vector& qkv, int pos, +bool Q4nxNpuForward::attention(const std::vector& qkv, int pos, const int* pos3, std::vector& ctx, int head_dim, int n_kv_heads, float theta, int rotary_dim, bool proportional, int window, @@ -4088,7 +4122,10 @@ bool Q4nxNpuForward::attention(const std::vector& qkv, int pos, // nothing per token. No-op unless the config declares rope_type "llama3". inv = q4nx_rope_llama3_inv_freq(inv, &rope_llama3_); } - const double ang = (double)pos * inv; + // Interleaved MRoPE: each rotary frequency rotates by its own axis's position (t, h or w). With the + // three planes equal this is exactly `pos * inv`, which is why text-only prompts are bit-identical + // to plain RoPE (measured: 0 angle difference over positions 0..262143, engine#1311/TQVL). + const double ang = (double)(pos3 ? pos3[mrope_plane(i)] : pos) * inv; cos_[i] = (float)std::cos(ang); sin_[i] = (float)std::sin(ang); } @@ -4413,7 +4450,7 @@ bool Q4nxNpuForward::step(int token, int pos, std::vector& logits) { } { LhScope _tlh(LH_ATTN); - if (!attention(qkv_use, pos, ctx, l_hd, l_nkv_heads, l_theta, l_rot, l_prop, + if (!attention(qkv_use, pos, nullptr, ctx, l_hd, l_nkv_heads, l_theta, l_rot, l_prop, l_win, kv_layer, !kv_shared)) { err_ = "attention failed"; return false; diff --git a/npu/forward/src/q4nx_semantics.c b/npu/forward/src/q4nx_semantics.c index 49fd44294..a11758a5b 100644 --- a/npu/forward/src/q4nx_semantics.c +++ b/npu/forward/src/q4nx_semantics.c @@ -229,6 +229,28 @@ int q4nx_semantics_from_config(const char* json_path, const char* model_dir, * picking one of the two. */ double d = 0; double prf_rope = 0; /* partial_rotary_factor found inside rope_parameters */ + /* The sections say WHICH rotary frequencies belong to the (t, h, w) axes; without them an interleaved-MRoPE + * consumer has nothing but a guess, so they are read next to the flag (both spellings) and a model that + * declares mrope without them is refused by the consumer rather than assumed. */ + { + const char* blocks[2] = {value_for_top(json, "rope_scaling"), value_for_top(json, "rope_parameters")}; + for (int b = 0; b < 2 && s->n_mrope_section == 0; b++) { + const char* blk = blocks[b]; + if (!blk || *blk != '{') continue; + const char* at = strstr(blk, "mrope_section"); + if (!at) continue; + at = strchr(at, '['); + if (!at) continue; + int got = 0; + const char* p2 = at + 1; + for (; p2 && *p2 && *p2 != ']' && got < 3; p2++) { + if (*p2 >= '0' && *p2 <= '9') { + s->mrope_section[got++] = (int)strtol(p2, (char**)&p2, 10); + } + } + s->n_mrope_section = got; + } + } /* Interleaved MRoPE is declarable in two spellings and only one was read here. FastFlowLM/OGA packs and * Qwen3.5 nest the rotary block under `rope_parameters` (the block below); a Hugging Face `qwen3_vl` config * (Qwen3-VL, engine #1311/TQVL) declares the same thing under `rope_scaling`. Reading only the first meant From 5a2738a0cdfa1d47bc3e51110c5ac71e2f465c93 Mon Sep 17 00:00:00 2001 From: pi-npufeat Date: Sun, 8 Nov 2026 12:30:22 -0400 Subject: [PATCH 04/10] npu: the decoder injects the vision tower's deepstack sets at the layers the pack names (refs #1311) Qwen3-VL's vision tower emits three extra feature sets from blocks 5/11/17 and the reference *adds* each to the decoder's hidden state at those layer indexes. Without that an image prompt cannot match the reference answer, no matter how exact the tower's own rows are. * vision_config.deepstack_visual_indexes is parsed (up to 8 entries) into the semantics; * step_batch_embeds() takes them as a HybridInputs bundle -- the three MRoPE position planes and sets x rows x H of deepstack features -- instead of growing the argument list again; * the batched body adds set s to the hidden state when the layer loop reaches deepstack_layers[s]; with no sets the pass is byte-identical to before; * a set count that disagrees with the pack is refused by name, and so is a null buffer. Measured with npu/forward/mrope_planes_probe on Qwen3-VL-2B's text container (151,936 logits per arm): pack declares 3 deepstack layers sets vs plain : mean |dlogit| 0.0829 host / 0.2894 device (they must differ) wrong set count : refused by name equal planes (control): byte-identical, 0 of 151,936 differ, host and device --- npu/forward/include/q4nx_forward.h | 29 +++++++++++++------ npu/forward/include/q4nx_semantics.h | 5 ++++ npu/forward/mrope_planes_probe.cpp | 42 +++++++++++++++++++++++++--- npu/forward/src/q4nx_forward.cpp | 35 +++++++++++++++++------ npu/forward/src/q4nx_semantics.c | 14 ++++++++++ 5 files changed, 105 insertions(+), 20 deletions(-) diff --git a/npu/forward/include/q4nx_forward.h b/npu/forward/include/q4nx_forward.h index a6a1a6ed9..ba1419ad5 100644 --- a/npu/forward/include/q4nx_forward.h +++ b/npu/forward/include/q4nx_forward.h @@ -148,14 +148,24 @@ class Q4nxNpuForward { // The per-token fallback path (MLA/MoE/GDN/... models) needs ids and is refused by name. bool step_batch_embeds(const std::vector& embeds, int start_pos, std::vector& logits); - // The same, with the three position planes an interleaved-MRoPE pack needs (engine#1311/TQVL, plan P4's - // `rope.planes`). `position_ids` is 3 x rows, plane-major (t, then h, then w) — exactly the layout the - // reference's `get_rope_index` produces and AMD's `text.onnx` takes. Empty means "the usual start_pos + row". - // Text rows have all three planes equal, which is bit-identical to plain RoPE (measured, 0 difference over - // positions 0..262143); image rows differ per axis and that difference is the whole point. A pack that - // declares mrope without sections is refused by name. - bool step_batch_embeds(const std::vector& embeds, const std::vector& position_ids, int start_pos, - std::vector& logits); + // What a vision-conditioned prompt adds on top of the embeddings (engine#1311/TQVL; plan P4's `rope.planes` + // and a per-layer feature injection): + // position_ids 3 x rows, plane-major (t, h, w) — the layout the reference's `get_rope_index` produces and + // AMD's `text.onnx` takes. Text rows have all three equal, which is bit-identical to plain RoPE + // (measured: 0 difference over positions 0..262143; image rows differ per axis and that is the + // whole point). A pack that declares mrope without sections is refused by name. + // deepstack sets x rows x H, one set per `vision_config.deepstack_visual_indexes` entry (Qwen3-VL: 5/11/17); + // set s is added to the hidden state when the layer loop reaches deepstack_layers[s]. This is how + // the reference injects the vision tower's extra features, and an answer cannot match it without. + struct HybridInputs { + const std::vector* position_ids = nullptr; + const float* deepstack = nullptr; + int deepstack_sets = 0; + }; + const int* deepstack_layers() const { return sem_.deepstack_layers; } + int n_deepstack_layers() const { return sem_.n_deepstack; } + bool step_batch_embeds(const std::vector& embeds, int start_pos, std::vector& logits, + const HybridInputs& in); // The same batched pass, but returning the logits of EVERY row (M x NV, row-major) instead of only // the last one. Speculative decoding needs them: a draft token is accepted only if the model's own @@ -365,6 +375,9 @@ class Q4nxNpuForward { // Set only while step_batch_embeds(..., position_ids, ...) runs: 3 x rows, plane-major, or null. const int* pos3_ = nullptr; int pos3_rows_ = 0; + // Set only while a step_batch_embeds() call carries deepstack sets: sets x rows x H. + const float* deepstack_ = nullptr; + int deepstack_rows_ = 0; // Which plane a rotary frequency belongs to under interleaved MRoPE (semantics' mrope_section, contiguous). int mrope_plane(int freq) const { if (sem_.n_mrope_section < 3) return 0; diff --git a/npu/forward/include/q4nx_semantics.h b/npu/forward/include/q4nx_semantics.h index 45c187e71..da7334da8 100644 --- a/npu/forward/include/q4nx_semantics.h +++ b/npu/forward/include/q4nx_semantics.h @@ -101,6 +101,11 @@ typedef struct { * guess a layout for. */ int mrope_section[3]; int n_mrope_section; + /* DEEPSTACK: which decoder layers take one of the vision tower's extra feature sets. Read from + * vision_config.deepstack_visual_indexes (Qwen3-VL: [5, 11, 17]); the consumer adds set s to the hidden state at + * layer deepstack_layers[s]. Empty means the pack has none. */ + int deepstack_layers[8]; + int n_deepstack; /* RoPE is configured PER LAYER TYPE (Gemma4 nests rope_parameters. * full_attention and .sliding_attention with different thetas and types). * Source: presence of rope_parameters.{full,sliding}_attention sub-blocks. */ diff --git a/npu/forward/mrope_planes_probe.cpp b/npu/forward/mrope_planes_probe.cpp index 55447c920..25a806411 100644 --- a/npu/forward/mrope_planes_probe.cpp +++ b/npu/forward/mrope_planes_probe.cpp @@ -58,8 +58,11 @@ int main(int argc, char** argv) { planes_img[rows + r] = r / 2; // h (a 2-wide image grid) planes_img[2 * rows + r] = r % 2; // w } - f.reset(); if (!f.step_batch_embeds(embeds, planes_eq, 0, eq)) { std::fprintf(stderr, "eq: %s\n", f.last_error()); return 1; } - f.reset(); if (!f.step_batch_embeds(embeds, planes_img, 0, diff)) { std::fprintf(stderr, "img: %s\n", f.last_error()); return 1; } + Q4nxNpuForward::HybridInputs in_eq, in_img; + in_eq.position_ids = &planes_eq; + in_img.position_ids = &planes_img; + f.reset(); if (!f.step_batch_embeds(embeds, 0, eq, in_eq)) { std::fprintf(stderr, "eq: %s\n", f.last_error()); return 1; } + f.reset(); if (!f.step_batch_embeds(embeds, 0, diff, in_img)) { std::fprintf(stderr, "img: %s\n", f.last_error()); return 1; } int eq_diff = 0; double img_delta = 0.0; for (size_t i = 0; i < plain.size(); i++) { @@ -70,6 +73,37 @@ int main(int argc, char** argv) { eq_diff, plain.size()); std::printf("image-style planes vs plain : mean |dlogit| %.4f (they must differ)\n", img_delta / double(plain.size())); const bool pass = eq_diff == 0 && img_delta > 0.0; - std::printf("%s\n", pass ? "PASS" : "FAIL"); - return pass ? 0 : 1; + // Deepstack: the pack's own layer list must be read (Qwen3-VL declares three), a set count that disagrees must be + // refused by name, and non-zero sets must move the logits -- the same control/effect shape as the planes above. + const int n_ds = f.n_deepstack_layers(); + std::printf("deepstack layers declared by the pack: %d\n", n_ds); + bool ds_ok = true; + if (n_ds > 0) { + const int Hd = int(embeds.size() / size_t(rows)); + std::vector sets(size_t(n_ds) * size_t(rows) * size_t(Hd), 0.05f); + Q4nxNpuForward::HybridInputs in_ds; + in_ds.deepstack = sets.data(); + in_ds.deepstack_sets = n_ds; + std::vector with_ds; + f.reset(); + if (!f.step_batch_embeds(embeds, 0, with_ds, in_ds)) { + std::fprintf(stderr, "deepstack: %s\n", f.last_error()); + return 1; + } + double ds_delta = 0.0; + for (size_t i = 0; i < plain.size(); i++) ds_delta += std::abs(double(plain[i]) - double(with_ds[i])); + ds_delta /= double(plain.size()); + std::printf("deepstack sets vs plain : mean |dlogit| %.4f (they must differ)\n", ds_delta); + Q4nxNpuForward::HybridInputs bad; + bad.deepstack = sets.data(); + bad.deepstack_sets = n_ds + 1; + std::vector refused; + f.reset(); + const bool ran = f.step_batch_embeds(embeds, 0, refused, bad); + std::printf("wrong set count : %s (%s)\n", ran ? "RAN" : "refused by name", f.last_error()); + ds_ok = ds_delta > 0.0 && !ran; + } + const bool all = pass && ds_ok; + std::printf("%s\n", all ? "PASS" : "FAIL"); + return all ? 0 : 1; } diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index e9c7468ad..8214a72ce 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -3024,12 +3024,7 @@ void Q4nxNpuForward::npu_gemm_rows(int role, const float* A, int M, int K, int N // The folded container's sign table: magic "HAD1", u32 count, count x (u32 width, u32 offset), // then the f32 signs those offsets index (the GGUF's prism.hadamard.sign_widths/sign_values). bool Q4nxNpuForward::step_batch_embeds(const std::vector& embeds, int start_pos, - std::vector& logits) { - return step_batch_embeds(embeds, std::vector(), start_pos, logits); -} - -bool Q4nxNpuForward::step_batch_embeds(const std::vector& embeds, const std::vector& position_ids, - int start_pos, std::vector& logits) { + std::vector& logits, const HybridInputs& in) { if (!ready_) { err_ = "not initialised"; return false; } const size_t H = (size_t)c_.H; if (H == 0 || embeds.empty() || embeds.size() % H != 0) { @@ -3040,6 +3035,7 @@ bool Q4nxNpuForward::step_batch_embeds(const std::vector& embeds, const s for (float v : embeds) if (!std::isfinite(v)) { err_ = "step_batch_embeds: a row holds a non-finite value"; return false; } const int rows = int(embeds.size() / H); + const std::vector& position_ids = in.position_ids ? *in.position_ids : std::vector(); if (!position_ids.empty()) { if (!sem_.mrope) { err_ = "step_batch_embeds: position_ids given but this pack does not declare interleaved MRoPE"; @@ -3059,16 +3055,28 @@ bool Q4nxNpuForward::step_batch_embeds(const std::vector& embeds, const s // The body indexes tokens[m] only when nothing is injected, but it still needs the row count and the guards // above it read tokens; a zero vector of the right length keeps every check honest (0 is a valid id) while // the fill itself takes the injected rows. + if (in.deepstack_sets > 0) { + if (in.deepstack_sets != sem_.n_deepstack) { + err_ = "step_batch_embeds: " + std::to_string(in.deepstack_sets) + " deepstack set(s) given but the pack " + "declares " + std::to_string(sem_.n_deepstack) + " deepstack layer(s)"; + return false; + } + if (!in.deepstack) { err_ = "step_batch_embeds: deepstack_sets without a buffer"; return false; } + } const std::vector placeholder(size_t(rows), 0); struct Restore { const float*& p; const int*& q; int& n; - ~Restore() { p = nullptr; q = nullptr; n = 0; } - } restore{inject_embeds_, pos3_, pos3_rows_}; + const float*& d; + int& dn; + ~Restore() { p = nullptr; q = nullptr; n = 0; d = nullptr; dn = 0; } + } restore{inject_embeds_, pos3_, pos3_rows_, deepstack_, deepstack_rows_}; inject_embeds_ = embeds.data(); pos3_ = position_ids.empty() ? nullptr : position_ids.data(); pos3_rows_ = rows; + deepstack_ = in.deepstack; + deepstack_rows_ = rows; return step_batch(placeholder, start_pos, logits); } @@ -3220,6 +3228,17 @@ bool Q4nxNpuForward::step_batch(const std::vector& tokens, int start_pos, std::vector QKV, CTX, O, ACT, HH, D, qkv_row, ctx_row; for (int l = 0; l < c_.NL; l++) { cur_layer_ = l; + // Deepstack (Qwen3-VL's vision tower emits three extra sets, added at these layers): the reference adds set + // s to the hidden state before the layer runs. With no sets (text-only, or the control arm) this is skipped + // and the pass is byte-identical to before. + if (deepstack_ && sem_.n_deepstack > 0) { + for (int s2 = 0; s2 < sem_.n_deepstack; s2++) { + if (sem_.deepstack_layers[s2] != l) continue; + const float* add = deepstack_ + size_t(s2) * size_t(deepstack_rows_) * size_t(H); + for (size_t i = 0; i < X.size(); i++) X[i] += add[i]; + break; + } + } const bool cached = weight_cached(0); const int lt0 = (l < sem_.n_layer_types) ? sem_.layer_types[l] : Q4NX_LAYER_FULL; // A GDN layer's host parameters (conv window, alpha/beta, ssm_a/dt, gated diff --git a/npu/forward/src/q4nx_semantics.c b/npu/forward/src/q4nx_semantics.c index a11758a5b..655b83a65 100644 --- a/npu/forward/src/q4nx_semantics.c +++ b/npu/forward/src/q4nx_semantics.c @@ -229,6 +229,20 @@ int q4nx_semantics_from_config(const char* json_path, const char* model_dir, * picking one of the two. */ double d = 0; double prf_rope = 0; /* partial_rotary_factor found inside rope_parameters */ + /* Deepstack layers: the vision tower emits extra feature sets that the decoder adds at these layer indexes. */ + { + const char* vc = value_for_top(json, "vision_config"); + const char* at = vc ? strstr(vc, "deepstack_visual_indexes") : nullptr; + if (at) at = strchr(at, '['); + if (at) { + int got = 0; + const char* p3 = at + 1; + for (; p3 && *p3 && *p3 != ']' && got < 8; p3++) { + if (*p3 >= '0' && *p3 <= '9') s->deepstack_layers[got++] = (int)strtol(p3, (char**)&p3, 10); + } + s->n_deepstack = got; + } + } /* The sections say WHICH rotary frequencies belong to the (t, h, w) axes; without them an interleaved-MRoPE * consumer has nothing but a guess, so they are read next to the flag (both spellings) and a model that * declares mrope without them is refused by the consumer rather than assumed. */ From af3240bb212dc00776c1237d006fdc096dceefc6 Mon Sep 17 00:00:00 2001 From: pi-npufeat Date: Sun, 8 Nov 2026 13:10:05 -0400 Subject: [PATCH 05/10] npu: deepstack sets are added at the image span only (a real OOB read in the first version), + the hybrid prompt probe Two things, and the first is a defect the wiring found immediately: * deepstack injection added set s to EVERY row of the prompt while the set has only rows_img rows -- so the loop read past the buffer (and, where it did not, added vision features to text positions). HybridInputs now carries the image row span and the injection runs there; a missing or out-of-range span is refused by name. * npu/forward/hybrid_prompt_probe.cpp: the engine side of a vision-conditioned prompt -- reads the token ids, the image rows' embeddings, the 3-plane positions and the deepstack sets (raw files + meta.txt from the vision graph and the tokenizer), runs the prefill through step_batch_embeds() and greedily decodes N tokens. Rows: the wiring runs (219 tokens, 196 image rows, 3 deepstack sets, host-GEMM and the layout checks pass), and the answer is NOT yet the reference's -- recorded on the PR rather than claimed: the first two attempts emitted <|im_end|> immediately, one of them because the hand-built prompt lacked the chat template's 'assistant\n' turn start. Gate 2's remaining work is the preprocessing/layout debugging loop, not more plumbing. --- CMakeLists.txt | 6 ++ npu/forward/hybrid_prompt_probe.cpp | 142 ++++++++++++++++++++++++++++ npu/forward/include/q4nx_forward.h | 6 ++ npu/forward/src/q4nx_forward.cpp | 22 ++++- 4 files changed, 172 insertions(+), 4 deletions(-) create mode 100644 npu/forward/hybrid_prompt_probe.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index f7ddfb368..5fe62637f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1269,6 +1269,12 @@ if(ONEBIT_NPU) # model's own table must give byte-identical logits to the token-id path, on the host and on the array. # The three-plane (MRoPE) position input: equal planes must be byte-identical to plain RoPE, different planes # must move the logits (npu/forward/mrope_planes_probe.cpp). + # The wiring a vision-conditioned prompt needs on the engine side: vision rows + 3-plane positions + deepstack + # sets through step_batch_embeds(), then a greedy decode (npu/forward/hybrid_prompt_probe.cpp). + add_executable(hybrid_prompt_probe npu/forward/hybrid_prompt_probe.cpp) + target_include_directories(hybrid_prompt_probe PRIVATE ${CMAKE_SOURCE_DIR}/npu + ${CMAKE_SOURCE_DIR}/npu/forward/include ${CMAKE_SOURCE_DIR}/npu/forward/engine-src) + target_link_libraries(hybrid_prompt_probe PRIVATE onebit_npu_forward onebit_npu_rt) add_executable(mrope_planes_probe npu/forward/mrope_planes_probe.cpp) target_include_directories(mrope_planes_probe PRIVATE ${CMAKE_SOURCE_DIR}/npu ${CMAKE_SOURCE_DIR}/npu/forward/include ${CMAKE_SOURCE_DIR}/npu/forward/engine-src) diff --git a/npu/forward/hybrid_prompt_probe.cpp b/npu/forward/hybrid_prompt_probe.cpp new file mode 100644 index 000000000..15412ba54 --- /dev/null +++ b/npu/forward/hybrid_prompt_probe.cpp @@ -0,0 +1,142 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// A vision-conditioned prompt through the decoder (engine#1311/TQVL): the wiring gate 2 needs, on the engine side. +// +// Reads what the vision half produced -- the prompt's token ids, the image rows' embeddings, the 3-plane positions and +// the deepstack sets -- and runs the prefill through step_batch_embeds(), then greedily decodes N tokens. Text rows' +// embeddings come from the model's own table (embed_row), so the only foreign rows are the image ones. +// +// hybrid_prompt_probe [--decode N] +// +// holds ids.i32, positions.i32 (3 x rows), pooled.f32 (rows_img x H), deepstack.f32 (sets x rows_img x H) +// and meta.json {"image_lo": i, "image_hi": j}. Build them with the vision graph (tools/export_qwen3vl_vision.py) +// plus the tokenizer; this probe does not touch a tokenizer. +#include "q4nx_forward.h" + +#include +#include +#include +#include +#include +#include +#include + +template +static bool read_vec(const std::string& path, std::vector& out) { + std::ifstream f(path, std::ios::binary); + if (!f) return false; + f.seekg(0, std::ios::end); + const auto n = (size_t)f.tellg() / sizeof(T); + f.seekg(0); + out.resize(n); + f.read((char*)out.data(), (std::streamsize)(n * sizeof(T))); + return (bool)f; +} + +int main(int argc, char** argv) { + if (argc < 4) { + std::fprintf(stderr, "usage: %s [--decode N]\n", argv[0]); + return 2; + } + int decode = 8; + for (int i = 4; i + 1 < argc; i++) + if (!std::strcmp(argv[i], "--decode")) decode = std::atoi(argv[i + 1]); + const std::string dir = argv[3]; + + Q4nxNpuForward f; + std::unique_ptr dev; + bool ok; + if (getenv("NPU_INFER_HOST_GEMM")) { + ok = f.init_host(argv[1], argv[2]); + } else { + dev = std::make_unique(0); + ok = f.init(*dev, argv[1], argv[2]); + } + if (!ok) { std::fprintf(stderr, "init: %s\n", f.last_error()); return 1; } + const int H = 0; // read from a row, below + (void)H; + + std::vector ids, positions; + std::vector pooled, deepstack; + if (!read_vec(dir + "/ids.i32", ids) || !read_vec(dir + "/positions.i32", positions) || + !read_vec(dir + "/pooled.f32", pooled) || !read_vec(dir + "/deepstack.f32", deepstack)) { + std::fprintf(stderr, "cannot read the input files from %s\n", dir.c_str()); + return 2; + } + // meta.json is tiny; the two numbers are read by position so the probe needs no JSON library + int meta[2] = {-1, -1}; + { + std::ifstream m(dir + "/meta.txt"); + if (m) m >> meta[0] >> meta[1]; // image_lo image_hi (one line each is fine) + } + if (meta[0] < 0 || meta[1] <= meta[0]) { std::fprintf(stderr, "bad meta (image_lo image_hi)\n"); return 2; } + const int rows = (int)ids.size(); + std::vector one; + f.embed_row(11, one); + const size_t Hd = one.size(); + const size_t rows_img = size_t(meta[1] - meta[0]); + if (pooled.size() != rows_img * Hd || positions.size() != size_t(3) * size_t(rows)) { + std::fprintf(stderr, "shapes: pooled %zu (want %zu), positions %zu (want %zu)\n", pooled.size() / Hd, + rows_img, positions.size(), size_t(3) * rows); + return 2; + } + const int n_ds = f.n_deepstack_layers(); + if (n_ds > 0 && deepstack.size() != size_t(n_ds) * rows_img * Hd) { + std::fprintf(stderr, "deepstack %zu, want %d sets x %zu rows x %zu\n", deepstack.size(), n_ds, rows_img, Hd); + return 2; + } + + // embedding rows: the model's own for text positions, the vision graph's for the image span + std::vector embeds; + embeds.reserve(size_t(rows) * Hd); + for (int r = 0; r < rows; r++) { + if (r >= meta[0] && r < meta[1]) { + const float* src = pooled.data() + size_t(r - meta[0]) * Hd; + embeds.insert(embeds.end(), src, src + Hd); + } else { + f.embed_row(ids[size_t(r)], one); + embeds.insert(embeds.end(), one.begin(), one.end()); + } + } + Q4nxNpuForward::HybridInputs in; + in.position_ids = &positions; + in.deepstack = n_ds > 0 ? deepstack.data() : nullptr; + in.deepstack_sets = n_ds; + in.image_lo = meta[0]; + in.image_hi = meta[1]; + + std::vector logits; + f.reset(); + if (!f.step_batch_embeds(embeds, 0, logits, in)) { std::fprintf(stderr, "prefill: %s\n", f.last_error()); return 1; } + auto argmax = [](const std::vector& v) { + int b = 0; + for (size_t i = 1; i < v.size(); i++) + if (v[i] > v[(size_t)b]) b = (int)i; + return b; + }; + std::vector out; + out.push_back(argmax(logits)); + int pos = int(positions[0 * rows + rows - 1]) + 1; // the generation step's position (delta already applied) + for (int i = 1; i < decode; i++) { + if (!f.step(out.back(), pos + i - 1, logits)) { std::fprintf(stderr, "step: %s\n", f.last_error()); return 1; } + out.push_back(argmax(logits)); + } + std::printf("prompt %d tokens (%zu image rows), deepstack %d set(s)\n", rows, rows_img, in.deepstack_sets); + std::printf("generated %d token ids:", (int)out.size()); + for (int t : out) std::printf(" %d", t); + std::printf("\n"); + return 0; +} diff --git a/npu/forward/include/q4nx_forward.h b/npu/forward/include/q4nx_forward.h index ba1419ad5..49ffd22c6 100644 --- a/npu/forward/include/q4nx_forward.h +++ b/npu/forward/include/q4nx_forward.h @@ -161,6 +161,11 @@ class Q4nxNpuForward { const std::vector* position_ids = nullptr; const float* deepstack = nullptr; int deepstack_sets = 0; + // The rows the deepstack sets belong to: the image span. They are added THERE, not everywhere -- a set has + // rows_img rows, so adding it to every row of the prompt would both be the wrong function and read past the + // buffer (the first version of this did exactly that and produced garbage tokens). + int image_lo = -1; + int image_hi = -1; }; const int* deepstack_layers() const { return sem_.deepstack_layers; } int n_deepstack_layers() const { return sem_.n_deepstack; } @@ -378,6 +383,7 @@ class Q4nxNpuForward { // Set only while a step_batch_embeds() call carries deepstack sets: sets x rows x H. const float* deepstack_ = nullptr; int deepstack_rows_ = 0; + int deepstack_lo_ = -1, deepstack_hi_ = -1; // Which plane a rotary frequency belongs to under interleaved MRoPE (semantics' mrope_section, contiguous). int mrope_plane(int freq) const { if (sem_.n_mrope_section < 3) return 0; diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 8214a72ce..22fa17e87 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -3062,6 +3062,11 @@ bool Q4nxNpuForward::step_batch_embeds(const std::vector& embeds, int sta return false; } if (!in.deepstack) { err_ = "step_batch_embeds: deepstack_sets without a buffer"; return false; } + if (in.image_lo < 0 || in.image_hi <= in.image_lo || in.image_hi > rows) { + err_ = "step_batch_embeds: deepstack sets need the image row span (image_lo/image_hi inside 0.." + + std::to_string(rows) + ")"; + return false; + } } const std::vector placeholder(size_t(rows), 0); struct Restore { @@ -3070,13 +3075,17 @@ bool Q4nxNpuForward::step_batch_embeds(const std::vector& embeds, int sta int& n; const float*& d; int& dn; - ~Restore() { p = nullptr; q = nullptr; n = 0; d = nullptr; dn = 0; } - } restore{inject_embeds_, pos3_, pos3_rows_, deepstack_, deepstack_rows_}; + int& lo; + int& hi; + ~Restore() { p = nullptr; q = nullptr; n = 0; d = nullptr; dn = 0; lo = -1; hi = -1; } + } restore{inject_embeds_, pos3_, pos3_rows_, deepstack_, deepstack_rows_, deepstack_lo_, deepstack_hi_}; inject_embeds_ = embeds.data(); pos3_ = position_ids.empty() ? nullptr : position_ids.data(); pos3_rows_ = rows; deepstack_ = in.deepstack; - deepstack_rows_ = rows; + deepstack_rows_ = in.image_hi - in.image_lo; + deepstack_lo_ = in.image_lo; + deepstack_hi_ = in.image_hi; return step_batch(placeholder, start_pos, logits); } @@ -3234,8 +3243,13 @@ bool Q4nxNpuForward::step_batch(const std::vector& tokens, int start_pos, if (deepstack_ && sem_.n_deepstack > 0) { for (int s2 = 0; s2 < sem_.n_deepstack; s2++) { if (sem_.deepstack_layers[s2] != l) continue; + // Only the image rows: a set has rows_img rows, and the reference adds it at the image-token + // positions, not across the prompt. const float* add = deepstack_ + size_t(s2) * size_t(deepstack_rows_) * size_t(H); - for (size_t i = 0; i < X.size(); i++) X[i] += add[i]; + const int lo = std::max(0, deepstack_lo_), hi = std::min((int)M, deepstack_hi_); + for (int r = lo; r < hi; r++) + for (size_t k = 0; k < (size_t)H; k++) + X[size_t(r) * (size_t)H + k] += add[size_t(r - lo) * (size_t)H + k]; break; } } From 010c2d04081af7fb789d337137075d42d6751ad6 Mon Sep 17 00:00:00 2001 From: pi-npufeat Date: Sun, 8 Nov 2026 14:07:02 -0400 Subject: [PATCH 06/10] npu: the hybrid probe takes a text-only span (the control that localises a bad image answer) image_lo == image_hi is now legal and means 'no image rows': no deepstack sets, no vision rows, the same prompt and the decode loop. That control is what separated the two halves of a wrong answer this session: the text-only prompt through the same probe decodes coherently while the image prompt stops at once, so the plumbing (prompt, positions, decode) is sound and the gap is in the image path. And the image path's graph is not the suspect either: on the real test image, our ONNX vision graph against the HF tower's own output on the SAME pixel_values gives pooled cosine 1.00000000 and deepstack 1.00000012/1.00000012/0.99999988 -- the export is exact for this image too, so what remains is the placement: the image rows' positions or their ordering. --- npu/forward/hybrid_prompt_probe.cpp | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/npu/forward/hybrid_prompt_probe.cpp b/npu/forward/hybrid_prompt_probe.cpp index 15412ba54..7b0600857 100644 --- a/npu/forward/hybrid_prompt_probe.cpp +++ b/npu/forward/hybrid_prompt_probe.cpp @@ -82,7 +82,8 @@ int main(int argc, char** argv) { std::ifstream m(dir + "/meta.txt"); if (m) m >> meta[0] >> meta[1]; // image_lo image_hi (one line each is fine) } - if (meta[0] < 0 || meta[1] <= meta[0]) { std::fprintf(stderr, "bad meta (image_lo image_hi)\n"); return 2; } + // image_lo == image_hi is the text-only control: no image rows, nothing to inject. + if (meta[0] < 0 || meta[1] < meta[0]) { std::fprintf(stderr, "bad meta (image_lo image_hi)\n"); return 2; } const int rows = (int)ids.size(); std::vector one; f.embed_row(11, one); @@ -113,10 +114,13 @@ int main(int argc, char** argv) { } Q4nxNpuForward::HybridInputs in; in.position_ids = &positions; - in.deepstack = n_ds > 0 ? deepstack.data() : nullptr; - in.deepstack_sets = n_ds; - in.image_lo = meta[0]; - in.image_hi = meta[1]; + // A text-only prompt (the control) has no image rows, so there is no span to add deepstack sets at. + if (n_ds > 0 && rows_img > 0) { + in.deepstack = deepstack.data(); + in.deepstack_sets = n_ds; + in.image_lo = meta[0]; + in.image_hi = meta[1]; + } std::vector logits; f.reset(); From 9ce309f8d24a0162f2575b9a500d4840dc5ce4aa Mon Sep 17 00:00:00 2001 From: pi-npufeat Date: Sun, 8 Nov 2026 15:06:56 -0400 Subject: [PATCH 07/10] tools/npu: a streaming QDQ int8 path for the vision graph (6.4 GiB peak, not 64) ORT's quantize_static needs ~64 GiB peak for this 1.6 GB graph -- measured, not estimated: a systemd scope recorded "64.1G memory peak" before the OOM killer took it, twice, on a box where other lanes hold most of the 122 GiB. This adds tools/vision_qdq.py and "quantize --stream", which does the same A8W8 job by editing the graph: per-output-channel int8 weights and per-tensor int8 activations as QDQ pairs, with activation ranges calibrated by running the fp32 graph once with every Gemm input exposed as an output (no intermediate-probing library needed). Measured, one weight tensor in memory at a time, under a scope capped at MemoryMax=16G: peak RSS 6.59 GB (vs the 64.1 G ORT needed) weights 1,628 MB -> 421 MB (3.9x) 104 weights, 104 activations Two bugs found on the way, both worth the comment they now carry: * QuantizeLinear's output type comes from its zero point and defaults to UINT8, which silently clamps every negative activation to 0: 103 of 104 activation tensors diverged from the fp32 graph (cosine -0.03). An int8 zero point fixes the type, and the same graph then lands at 0.61-0.91. * save_model with external data APPENDS to an existing .data file, so a second run wrote an 841 MB blob for 421 MB of weights. The stale pair is now unlinked first. Accuracy, honestly: weights-only (W8A32) gives cosine 0.9937 against the fp32 graph, and full A8W8 gives 0.61-0.91 against the HF tower -- below this repo's 0.999 per-patch bar. Per-K activation scales would close some of that gap but cannot fold into our EP's int32 accumulation (sA would vary along K), so a static per-tensor activation scale is the only form the device path can use. The choice between W8A32 at 0.9937 and an A8W8 arm judged by its answers is an owner call, not one to make silently here. --- tools/export_qwen3vl_vision.py | 17 ++- tools/vision_qdq.py | 236 +++++++++++++++++++++++++++++++++ 2 files changed, 251 insertions(+), 2 deletions(-) create mode 100644 tools/vision_qdq.py diff --git a/tools/export_qwen3vl_vision.py b/tools/export_qwen3vl_vision.py index 902e544ed..47a1ef63d 100755 --- a/tools/export_qwen3vl_vision.py +++ b/tools/export_qwen3vl_vision.py @@ -260,13 +260,19 @@ def bench(out_dir: str, images: int, providers: str): return 0 -def quantize(fp32_dir: str, out_dir: str, calib: int): +def quantize(fp32_dir: str, out_dir: str, calib: int, stream: bool = False, clip_percentile: float = 99.9): """QDQ int8, A8W8 per-channel -- the shape our ORT EP runs (Quark-style A8W8). Activations are quantised per tensor (QDQ), weights per output channel, both int8, and `image_grid_thw` is left alone (it is an integer input, not an activation). The calibration set is deterministic images at the exported bucket, so the row is reproducible; a real deployment would calibrate on its own data. """ + if stream: + import vision_qdq + + return vision_qdq.quantise(os.path.join(fp32_dir, "vision_fp32.onnx"), os.path.join(out_dir, "vision_qdq.onnx"), + calib=calib, clip_percentile=clip_percentile) + import onnxruntime as ort from onnxruntime.quantization import CalibrationDataReader, CalibrationMethod, QuantFormat, QuantType, quantize_static from onnxruntime.quantization.shape_inference import quant_pre_process # kept for the record above @@ -341,6 +347,12 @@ def main(): q.add_argument("fp32_dir") q.add_argument("out_dir") q.add_argument("--calib", type=int, default=32) + q.add_argument("--stream", action="store_true", + help="streaming per-tensor QDQ instead of ORT's quantize_static (which needs ~64 GiB peak for this " + "graph: measured by a systemd scope at 64.1 G before the OOM killer; the streaming path peaks " + "at 6.4 G)") + q.add_argument("--clip-percentile", type=float, default=99.9, + help="with --stream: per-tensor |x| percentile used as the activation range") b = sub.add_parser("bench") b.add_argument("out_dir") b.add_argument("--images", type=int, default=3) @@ -355,7 +367,8 @@ def main(): export(a.hf_dir, a.out_dir, a.opset) return 0 if a.cmd == "quantize": - return quantize(a.fp32_dir, a.out_dir, a.calib) + quantize(a.fp32_dir, a.out_dir, a.calib, a.stream, a.clip_percentile) + return 0 if a.cmd == "bench": return bench(a.out_dir, a.images, a.providers) return check(a.hf_dir, a.out_dir, a.images, a.onnx) diff --git a/tools/vision_qdq.py b/tools/vision_qdq.py new file mode 100644 index 000000000..069b37833 --- /dev/null +++ b/tools/vision_qdq.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +"""Streaming QDQ A8W8 for the vision graph: one weight tensor in memory at a time. + +Used by `export_qwen3vl_vision.py quantize --stream`. + +ORT's quantize_static needs ~64 GiB peak for this 1.6 GB graph (measured: the systemd scope recorded 64.1 G peak before +the OOM killer), which a shared box cannot spare. This does the same job by editing the graph directly: + + * weights: per-output-channel symmetric int8, emitted as QuantizeLinear -> DequantizeLinear pairs; + * activations: per-tensor symmetric int8 before each MatMul, with the scale calibrated from a few images (the fp32 + graph is run once with every MatMul input exposed as an output, so no intermediate probing library is needed); + * everything else (Conv, norms, softmax, split) stays float. + + qdq_stream.py [--calib N] +""" +import argparse +import json +import os +import sys + +import numpy as np +import onnx +from onnx import TensorProto, helper, numpy_helper + +sys.path.insert(0, "/tmp/qvl") + + +def matmul_weights(model): + """(node, initializer, per_output_axis) for every layer that multiplies by a WEIGHT initializer. + + The tower's projections are `Gemm(A, W, bias)` with `transB=1` (torch exports Linear that way), so W is [N, K] and + its per-output-channel scale runs along axis 0. The `MatMul` nodes that remain are attention's A@K^T and A@V, + where *both* operands are activations -- quantising those needs both inputs quantised and is deliberately left + float here, since the projections carry the FLOPs. + """ + inits = {i.name: i for i in model.graph.initializer} + out = [] + for n in model.graph.node: + if n.op_type not in ("Gemm", "MatMul") or len(n.input) < 2 or n.input[1] not in inits: + continue + w = numpy_helper.to_array(inits[n.input[1]]) + if w.ndim != 2: + continue + trans_b = 0 + if n.op_type == "Gemm": + for at in n.attribute: + if at.name == "transB": + trans_b = int(at.i) + # reduce over K: axis 1 for W [N, K] (Gemm transB=1, the torch Linear export), axis 0 for W [K, N] + out.append((n, inits[n.input[1]], 1 if (n.op_type == "Gemm" and trans_b == 1) else 0)) + return out, inits + + +def expose_outputs(model, names): + have = {o.name for o in model.graph.output} + for nm in names: + if nm in have: + continue + model.graph.output.append(helper.make_tensor_value_info(nm, TensorProto.FLOAT, None)) + + +def run_calibration(path, names, images, bucket, patch, temporal, real_image="", calib_pv="", clip_pct=99.9): + import onnxruntime as ort + # the same deterministic image generator the tool's check rows use, copied here so the streaming quantiser does not + # depend on the harness script + from hybrid_input import image_rows # noqa: F401 (the real image path is exercised by the check rows) + def make_image(rng, gh, gw, patch, temporal): + H, W = gh * patch, gw * patch + yy, xx = np.mgrid[0:H, 0:W] + img = np.zeros((3, H, W), dtype=np.float32) + img[0] = (yy / max(H - 1, 1)) * 0.8 + 0.1 + img[1] = (xx / max(W - 1, 1)) * 0.8 + 0.1 + img[2] = ((yy // 32 + xx // 32) % 2) * 0.7 + 0.1 + img += rng.normal(0, 0.02, img.shape).astype(np.float32) + img = np.clip(img, 0, 1) + blocks = img.reshape(3, gh, patch, gw, patch).transpose(1, 3, 0, 2, 4) + if temporal != 1: + blocks = np.repeat(blocks[:, :, None], temporal, axis=2) + return blocks.reshape(gh * gw, 3 * temporal * patch * patch).astype(np.float32) + + sess = ort.InferenceSession(path, providers=["CPUExecutionProvider"]) + got = {nm: 0.0 for nm in names} # the max over images of each image's p99.9 |x| (see --clip-percentile) + feeds = [] + if calib_pv: # the real picture first: synthetic gradients do not bound a photo + feeds.append(np.load(calib_pv).astype(np.float32)) + elif real_image: + feeds.append(image_rows(real_image)) + rng = np.random.default_rng(3) + for _ in range(images): + feeds.append(make_image(rng, bucket, bucket, patch, temporal)) + for pv in feeds: + grid = np.array([[1, bucket, bucket]], dtype=np.int64) + outs = sess.run(names, {"pixel_values": pv, "image_grid_thw": grid}) + for nm, arr in zip(names, outs): + # a single outlier must not set the whole tensor's scale: one activation here has absmax 1083 while the + # mass of that tensor sits near zero, and max-based scales zero it out (measured: 103/104 tensors diverge, + # cosine -0.03). A high percentile keeps the dynamic range the mass actually uses. + a_ = np.abs(arr.reshape(-1)).astype(np.float32) + got[nm] = max(got[nm], float(np.percentile(a_, clip_pct))) + return got + + +def quantise(src, dst, calib=6, clip_percentile=99.9, calib_pv="", bucket=28, patch=16, temporal=2): + """Quantise `src` into `dst` (+ `dst`.data). Returns the number of quantised weights.""" + a = argparse.Namespace(src=src, dst=dst, calib=calib, bucket=bucket, patch=patch, temporal=temporal, + no_act_qdq=False, real_image="", calib_pv=calib_pv, clip_percentile=clip_percentile) + return _run(a) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("src") + ap.add_argument("dst") + ap.add_argument("--calib", type=int, default=6) + ap.add_argument("--no-act-qdq", action="store_true", help="weights only (W8A32): isolates which side loses the answer") + ap.add_argument("--bucket", type=int, default=28) + ap.add_argument("--patch", type=int, default=16) + ap.add_argument("--temporal", type=int, default=2) + ap.add_argument("--real-image", default="", help="calibrate on this image too (needs PIL)") + ap.add_argument("--calib-pv", default="", help="calibrate on these pixels (a .npy), so no PIL is needed here") + ap.add_argument("--clip-percentile", type=float, default=99.9, + help="per-tensor |x| percentile used as the activation range (100 = plain max, which loses a tensor " + "to one outlier: measured 103/104 tensors diverge)") + return _run(ap.parse_args()) + + +def _run(a): + model = onnx.load(a.src) # ~2 GB in memory; the weights are not copied per tensor + pairs, inits = matmul_weights(model) + print(f"{len(pairs)} MatMul weights to quantise") + + # --- calibrate the activations: expose each MatMul's A input, run a few images, take the absmax + names = sorted({n.input[0] for n, _, _ in pairs}) + calib_model = onnx.load(a.src) + expose_outputs(calib_model, names) + cpath = a.dst + ".calib.onnx" + onnx.save(calib_model, cpath) + del calib_model + act_absmax = run_calibration(cpath, names, a.calib, a.bucket, a.patch, a.temporal, a.real_image, a.calib_pv, + a.clip_percentile) + os.unlink(cpath) + vals = sorted(act_absmax.values()) + tiny = [nm for nm, v in act_absmax.items() if v < 1e-3] + print(f"calibrated {len(act_absmax)} activation tensors on {a.calib} images: " + f"absmax min {vals[0]:.4g} median {vals[len(vals)//2]:.4g} max {vals[-1]:.4g}; " + f"{len(tiny)} below 1e-3" + (f" (e.g. {', '.join(tiny[:4])})" if tiny else "")) + + # --- rewrite: per-channel int8 weights, per-tensor int8 activations, QDQ pairs + new_inits, replaced = [], set() + act_scale = {} if a.no_act_qdq else {nm: max(v / 127.0, 1e-8) for nm, v in act_absmax.items()} + # QuantizeLinear's output type comes from its zero point, and its default is UINT8 -- which silently clamps every + # negative activation to 0. An int8 zero point makes the codes int8 and is what the DequantizeLinear below expects + # (measured: without it the first Gemm's activations lose all negatives, 103/104 tensors diverge, cosine -0.03). + for nm, sc in act_scale.items(): + new_inits.append(numpy_helper.from_array(np.array([sc], dtype=np.float32), name=nm + "_qdq_scale")) + new_inits.append(numpy_helper.from_array(np.zeros([1], dtype=np.int8), name=nm + "_qdq_zero")) + + # quantise each weight into a new initializer pair, remembering what each Gemm's inputs become + gemm_rewrite = {} + for n, init, axis in pairs: + w = numpy_helper.to_array(init) # one tensor at a time; the 64 GiB ORT needs for this is why + if w.ndim != 2 or 0 in w.shape: + continue + red = (1,) if axis == 1 else (0,) + cols = w.shape[0] if axis == 1 else w.shape[1] + amax = np.abs(w).max(axis=red, keepdims=True) + scale = np.maximum(amax / 127.0, 1e-8).astype(np.float32) + q = np.clip(np.rint(w / scale), -127, 127).astype(np.int8) + base = init.name + "_qdq" + new_inits.append(numpy_helper.from_array(q, name=base + "_q")) + new_inits.append(numpy_helper.from_array(scale.reshape(cols), name=base + "_s")) + # the codes keep the weight's layout, so the per-channel axis is the OTHER one (scale runs along N) + gemm_rewrite[id(n)] = (n, base, 1 - red[0]) + replaced.add(init.name) + del w, q, amax, scale + + # rebuild the node list in topological order, putting each QDQ pair immediately before its consumer + act_emitted = set() + out_nodes = [] + for n in model.graph.node: + rw = gemm_rewrite.get(id(n)) + if rw: + _, base, dq_axis = rw + a_name = n.input[0] + if a_name in act_scale and a_name not in act_emitted: + act_emitted.add(a_name) + out_nodes.append(helper.make_node("QuantizeLinear", + [a_name, a_name + "_qdq_scale", a_name + "_qdq_zero"], + [a_name + "_qdq_q"], name=a_name + "_qdq_ql")) + out_nodes.append(helper.make_node("DequantizeLinear", [a_name + "_qdq_q", a_name + "_qdq_scale"], + [a_name + "_qdq_dq"], name=a_name + "_qdq_dql")) + for n2, _, _ in pairs: + if n2.input[0] == a_name: + n2.input[0] = a_name + "_qdq_dq" + out_nodes.append(helper.make_node("DequantizeLinear", [base + "_q", base + "_s"], [base + "_dq"], + name=base, axis=dq_axis)) + for i, inp in enumerate(n.input): + if inp in replaced: + n.input[i] = base + "_dq" + out_nodes.append(n) + + keep = [i for i in model.graph.initializer if i.name not in replaced] + model.graph.ClearField("initializer") + model.graph.initializer.extend(keep + new_inits) + model.graph.ClearField("node") + model.graph.node.extend(out_nodes) + onnx.checker.check_model(model) + # a previous run's weight blob is appended to, not replaced, by save_model with external data + for stale in (a.dst, a.dst + ".data"): + if os.path.exists(stale): + os.unlink(stale) + onnx.save_model(model, a.dst, save_as_external_data=True, all_tensors_to_one_file=True, + location=os.path.basename(a.dst) + ".data", size_threshold=1024) + print(f"wrote {a.dst} ({os.path.getsize(a.dst) / 1e6:.1f} MB) + .data " + f"({os.path.getsize(a.dst + '.data') / 1e6:.1f} MB); {len(replaced)} weights, {len(act_emitted)} activations") + return len(replaced) + + +if __name__ == "__main__": + sys.exit(main()) From 09072deab86b2952dbda1d401e465f70d5ae4498 Mon Sep 17 00:00:00 2001 From: pi-npufeat Date: Sun, 8 Nov 2026 17:13:06 -0400 Subject: [PATCH 08/10] npu: interleaved MRoPE axes, DeepStack after layer l, and the tool's exit code Five of the six review items from the two CHANGES verdicts on engine #1498. (The sixth, a CI step that syntax-checks every npu/forward/src/*.c with gcc-16 -std=c11, is a workflow file and this app's token has no `workflow` scope, so it rides as a patch for the owner instead. The item was non-blocking.) Decoder: 1. `nullptr` in `q4nx_semantics.c` -- a C file that the CMake build happens to compile as C++ inside q4nx_mrope_test, npu_rmsnorm_offset_test, q4nx_layer_kinds_test and onebit_npu_forward, so a C++-only token reached a green build. `NULL`, plus the two parser overruns in the same file: both loops advanced the cursor with the `for` clause after `strtol` had already moved it, which stepped past the closing `]` and kept scanning until some later `]` appeared. The cursor is now moved by strtol alone. 2. `mrope_plane()` was a contiguous split of the three sections. The reference does not split contiguously: it builds cos/sin for all three planes and recomposes, overwriting every third frequency from the h and w planes over a stride-3 window of length section[dim] * 3 (transformers `Qwen3VLTextRotaryEmbedding.recomposition_frequencies`). So axis(i) = 1 if i % 3 == 1 && i < 3 * section[1] = 2 if i % 3 == 2 && i < 3 * section[2] = 0 otherwise At [24, 20, 20] that is 24 t / 20 h / 20 w, and it disagrees with the contiguous reading on **43 of the 64** frequencies (measured, not estimated; the reviewer's number was 43 too). The rule now lives in q4nx_semantics.h as `q4nx_mrope_axis()` -- a free C function -- and `mrope_plane()` is one line over it, so tests/q4nx_mrope_test.c pins the counts, both window edges and the 43/64 delta without building a forward pass. 3. DeepStack timing. Set s was added before text layer deepstack_visual_indexes[s] (8/16/24), but those indexes are the VISION tower's layers the sets are tapped from. The reference adds set `layer_idx` AFTER text decoder layer `layer_idx` runs, for layer_idx < number of sets (transformers Qwen3VLTextModel.forward: "add visual features to the hidden states of first several layers", `_deepstack_process` inside the loop after the call). Now `l < n_deepstack` after the layer. While moving it I deleted `cur_layer_ = l`, which silently made every layer read the last layer's id, and the text-only control's answer changed from "A person's face is partially obscured..." to garbage. The control caught it (the second time this session a known-good control caught a regression before a push), `cur_layer_` is back, and the control is byte-identical again: `32 1697 594 3579 374 25244 95701 553`. Tool (`quantize --stream`): 4. `tools/vision_qdq.py` imported its image generator from `/tmp/qvl` -- not in the tree, and first on sys.path for every lane on the box. The generator now lives in the module, the path insert is gone, and nothing outside the repo is imported. 5. `main()` discarded the ORT path's exit code, so a graph the tool itself calls "FAILS ... do not use it" exited 0. The rc is propagated, and `--stream` gets the same contract: it self-checks against the fp32 graph and returns 1 below the bar. Verified on this head: `self-check: worst cosine 0.606102 ... FAIL; the bar is 0.999`, rc=1. Also fixed: the duplicate copyright header, the stale `qdq_stream.py` usage line, and the "every MatMul input" wording (it is every `Gemm` input; attention's activation-times-activation MatMuls are deliberately float). Tests: q4nx_mrope_test, npu_rmsnorm_offset_test and q4nx_layer_kinds_test all pass, and the text-only control is unchanged. The image prompt is **still** bad (`32 23 <|im_end|> ...`) -- HF captions the same ids, positions and pooled features, so items 2 and 3 were real bugs against the reference but are not sufficient, and the next step is a logit diff at the first decode step rather than another placement search. --- npu/forward/include/q4nx_forward.h | 9 +-- npu/forward/include/q4nx_semantics.h | 23 ++++++++ npu/forward/src/q4nx_forward.cpp | 33 ++++++----- npu/forward/src/q4nx_semantics.c | 20 +++++-- tests/q4nx_mrope_test.c | 26 +++++++++ tools/export_qwen3vl_vision.py | 6 +- tools/vision_qdq.py | 85 +++++++++++++++++----------- 7 files changed, 140 insertions(+), 62 deletions(-) diff --git a/npu/forward/include/q4nx_forward.h b/npu/forward/include/q4nx_forward.h index 49ffd22c6..2b2ef0e2a 100644 --- a/npu/forward/include/q4nx_forward.h +++ b/npu/forward/include/q4nx_forward.h @@ -384,12 +384,9 @@ class Q4nxNpuForward { const float* deepstack_ = nullptr; int deepstack_rows_ = 0; int deepstack_lo_ = -1, deepstack_hi_ = -1; - // Which plane a rotary frequency belongs to under interleaved MRoPE (semantics' mrope_section, contiguous). - int mrope_plane(int freq) const { - if (sem_.n_mrope_section < 3) return 0; - const int a = sem_.mrope_section[0], b = sem_.mrope_section[0] + sem_.mrope_section[1]; - return freq < a ? 0 : (freq < b ? 1 : 2); - } + // Which plane a rotary frequency belongs to under interleaved MRoPE. The rule lives in q4nx_semantics.h as + // q4nx_mrope_axis() so the C unit test can pin it without building a forward pass. + int mrope_plane(int freq) const { return q4nx_mrope_axis(freq, sem_.n_mrope_section, sem_.mrope_section); } // Some models store the embedding QUANTIZED (Gemma4-E2B/E4B: dtype I8), and // reading it as bf16 yields ~zero - which invalidated the whole forward and // the tied LM head (all-zero logits). When the dtype is not BF16 the diff --git a/npu/forward/include/q4nx_semantics.h b/npu/forward/include/q4nx_semantics.h index da7334da8..0d43764b5 100644 --- a/npu/forward/include/q4nx_semantics.h +++ b/npu/forward/include/q4nx_semantics.h @@ -204,6 +204,29 @@ void q4nx_semantics_default(Q4nxSemantics* s); int q4nx_semantics_from_config(const char* json_path, const char* model_dir, Q4nxSemantics* s); +/* Which axis a rotary frequency belongs to under INTERLEAVED MRoPE. + * + * The three `mrope_section` values are not contiguous runs of frequencies. The reference builds cos/sin for all three + * planes and recomposes them: start from the t plane and, for each of h and w, overwrite every third frequency with that + * plane's value over a stride-3 window of length `section[dim] * 3` (transformers + * `Qwen3VLTextRotaryEmbedding.recomposition_frequencies`: `idx = slice(offset, mrope_section[dim] * 3, 3)`). + * + * axis(i) = 1 if i % 3 == 1 && i < 3 * section[1] + * = 2 if i % 3 == 2 && i < 3 * section[2] + * = 0 otherwise + * + * For [24, 20, 20] that is 24 t / 20 h / 20 w frequencies, 40 of the 64 in positions a contiguous split would not put + * them in -- which is why treating the sections as contiguous runs left every image prompt's answer bad while all + * equal-plane (text-only) prompts stayed exact. Frequency index is the index within head_dim/2. + */ +static inline int q4nx_mrope_axis(int freq, int n_section, const int* section) { + if (n_section < 3 || !section) return 0; + const int m = freq % 3; + if (m == 1 && freq < 3 * section[1]) return 1; + if (m == 2 && freq < 3 * section[2]) return 2; + return 0; +} + /* Human-readable one-line dump of every flag with its source field. */ void q4nx_semantics_dump(const Q4nxSemantics* s); diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 22fa17e87..001e73f8d 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -3236,23 +3236,7 @@ bool Q4nxNpuForward::step_batch(const std::vector& tokens, int start_pos, std::vector QKV, CTX, O, ACT, HH, D, qkv_row, ctx_row; for (int l = 0; l < c_.NL; l++) { - cur_layer_ = l; - // Deepstack (Qwen3-VL's vision tower emits three extra sets, added at these layers): the reference adds set - // s to the hidden state before the layer runs. With no sets (text-only, or the control arm) this is skipped - // and the pass is byte-identical to before. - if (deepstack_ && sem_.n_deepstack > 0) { - for (int s2 = 0; s2 < sem_.n_deepstack; s2++) { - if (sem_.deepstack_layers[s2] != l) continue; - // Only the image rows: a set has rows_img rows, and the reference adds it at the image-token - // positions, not across the prompt. - const float* add = deepstack_ + size_t(s2) * size_t(deepstack_rows_) * size_t(H); - const int lo = std::max(0, deepstack_lo_), hi = std::min((int)M, deepstack_hi_); - for (int r = lo; r < hi; r++) - for (size_t k = 0; k < (size_t)H; k++) - X[size_t(r) * (size_t)H + k] += add[size_t(r - lo) * (size_t)H + k]; - break; - } - } + cur_layer_ = l; // (removed with the old pre-layer deepstack block by accident; every layer read the last one) const bool cached = weight_cached(0); const int lt0 = (l < sem_.n_layer_types) ? sem_.layer_types[l] : Q4NX_LAYER_FULL; // A GDN layer's host parameters (conv window, alpha/beta, ssm_a/dt, gated @@ -3553,6 +3537,21 @@ bool Q4nxNpuForward::step_batch(const std::vector& tokens, int start_pos, fprintf(stderr, " [LB] m=%d l=%d %.6f %.4f\n", m, l, ms / H, mmx); } } + // Deepstack, in the reference's place: set `l` is added to the hidden state AFTER text decoder layer `l` + // runs, for l < number of sets (transformers Qwen3VLTextModel.forward: "add visual features to the hidden + // states of first several layers", `hidden_states = _deepstack_process(hidden_states, mask, sets[layer_idx])` + // inside the loop, after the call). The pack's `deepstack_visual_indexes` are the VISION tower's layers the + // sets are tapped from (5/11/17 for this model) and are not text layer indexes -- using them here put the + // three sets at text layers 8/16/24, before the layer, which is why the image prompts answered "A\n". + if (deepstack_ && l < sem_.n_deepstack) { + // Only the image rows: a set has rows_img rows, and the reference adds it at the image-token positions, + // not across the prompt. + const float* add = deepstack_ + size_t(l) * size_t(deepstack_rows_) * size_t(H); + const int lo = std::max(0, deepstack_lo_), hi = std::min((int)M, deepstack_hi_); + for (int r = lo; r < hi; r++) + for (size_t k = 0; k < (size_t)H; k++) + X[size_t(r) * (size_t)H + k] += add[size_t(r - lo) * (size_t)H + k]; + } } if (phb) { g_ph.layer = ph_ms(tp_lap); tp_lap = std::chrono::steady_clock::now(); } diff --git a/npu/forward/src/q4nx_semantics.c b/npu/forward/src/q4nx_semantics.c index 655b83a65..bfb88d276 100644 --- a/npu/forward/src/q4nx_semantics.c +++ b/npu/forward/src/q4nx_semantics.c @@ -232,13 +232,19 @@ int q4nx_semantics_from_config(const char* json_path, const char* model_dir, /* Deepstack layers: the vision tower emits extra feature sets that the decoder adds at these layer indexes. */ { const char* vc = value_for_top(json, "vision_config"); - const char* at = vc ? strstr(vc, "deepstack_visual_indexes") : nullptr; + const char* at = vc ? strstr(vc, "deepstack_visual_indexes") : NULL; if (at) at = strchr(at, '['); if (at) { int got = 0; const char* p3 = at + 1; - for (; p3 && *p3 && *p3 != ']' && got < 8; p3++) { - if (*p3 >= '0' && *p3 <= '9') s->deepstack_layers[got++] = (int)strtol(p3, (char**)&p3, 10); + while (*p3 && *p3 != ']' && got < 8) { + if (*p3 >= '0' && *p3 <= '9') { + char* end = NULL; + s->deepstack_layers[got++] = (int)strtol(p3, &end, 10); + p3 = end; /* strtol already stopped on the separator; a for-clause p3++ would skip the ']' */ + } else { + p3++; + } } s->n_deepstack = got; } @@ -257,9 +263,13 @@ int q4nx_semantics_from_config(const char* json_path, const char* model_dir, if (!at) continue; int got = 0; const char* p2 = at + 1; - for (; p2 && *p2 && *p2 != ']' && got < 3; p2++) { + while (*p2 && *p2 != ']' && got < 3) { if (*p2 >= '0' && *p2 <= '9') { - s->mrope_section[got++] = (int)strtol(p2, (char**)&p2, 10); + char* end = NULL; + s->mrope_section[got++] = (int)strtol(p2, &end, 10); + p2 = end; /* as above: let strtol move the cursor, not the loop */ + } else { + p2++; } } s->n_mrope_section = got; diff --git a/tests/q4nx_mrope_test.c b/tests/q4nx_mrope_test.c index 7fe900173..e8a42bfbf 100644 --- a/tests/q4nx_mrope_test.c +++ b/tests/q4nx_mrope_test.c @@ -97,6 +97,32 @@ int main(void) { q4nx_semantics_from_config(cfg, dir, &s); check(s.mrope == 0, "mrope_interleaved:false stays plain"); + /* 5. The interleaved axis rule (engine #1498). The sections are stride-3 windows, not contiguous runs: at + * [24, 20, 20] there must be 24 t, 20 h and 20 w frequencies out of 64, and the h/w windows must sit on + * i % 3 == 1 / 2 and stop at 3 * section[dim]. A contiguous split puts 40 of the 64 on the wrong axis. */ + { + int sec[3] = {24, 20, 20}; + int count[3] = {0, 0, 0}; + for (int i = 0; i < 64; i++) count[q4nx_mrope_axis(i, 3, sec)]++; + check(count[0] == 24 && count[1] == 20 && count[2] == 20, "interleaved [24,20,20] -> 24 t / 20 h / 20 w"); + check(q4nx_mrope_axis(1, 3, sec) == 1 && q4nx_mrope_axis(4, 3, sec) == 1, "i%3==1 is the h window"); + check(q4nx_mrope_axis(2, 3, sec) == 2 && q4nx_mrope_axis(5, 3, sec) == 2, "i%3==2 is the w window"); + check(q4nx_mrope_axis(0, 3, sec) == 0 && q4nx_mrope_axis(3, 3, sec) == 0, "i%3==0 stays t"); + check(q4nx_mrope_axis(60, 3, sec) == 0, "past 3 * section[1] the h window closes (i=60 is t)"); + check(q4nx_mrope_axis(61, 3, sec) == 0 && q4nx_mrope_axis(62, 3, sec) == 0, + "past 3 * section[2] the w window closes too"); + /* the contiguous mapping this replaces, for the record: 43 of 64 differ (measured, and the number the + * reviewer gave for engine #1498) */ + int diff = 0; + for (int i = 0; i < 64; i++) { + int old = i < 24 ? 0 : (i < 44 ? 1 : 2); + if (old != q4nx_mrope_axis(i, 3, sec)) diff++; + } + check(diff == 43, "the contiguous reading differs on 43 of 64 frequencies"); + /* and with no sections nothing rotates off the t plane */ + check(q4nx_mrope_axis(1, 0, sec) == 0, "no mrope_section -> plain (t) axis"); + } + if (failures) { printf("q4nx_mrope: %d failure(s)\n", failures); return 1; } printf("q4nx_mrope: ok\n"); return 0; diff --git a/tools/export_qwen3vl_vision.py b/tools/export_qwen3vl_vision.py index 47a1ef63d..7ae2b52f1 100755 --- a/tools/export_qwen3vl_vision.py +++ b/tools/export_qwen3vl_vision.py @@ -268,8 +268,11 @@ def quantize(fp32_dir: str, out_dir: str, calib: int, stream: bool = False, clip bucket, so the row is reproducible; a real deployment would calibrate on its own data. """ if stream: + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # tools/vision_qdq.py sits beside this file import vision_qdq + # same contract as the ORT path below: 1 when the quantised graph misses the cosine bar, so a failing graph can + # never exit 0 return vision_qdq.quantise(os.path.join(fp32_dir, "vision_fp32.onnx"), os.path.join(out_dir, "vision_qdq.onnx"), calib=calib, clip_percentile=clip_percentile) @@ -367,8 +370,7 @@ def main(): export(a.hf_dir, a.out_dir, a.opset) return 0 if a.cmd == "quantize": - quantize(a.fp32_dir, a.out_dir, a.calib, a.stream, a.clip_percentile) - return 0 + return quantize(a.fp32_dir, a.out_dir, a.calib, a.stream, a.clip_percentile) if a.cmd == "bench": return bench(a.out_dir, a.images, a.providers) return check(a.hf_dir, a.out_dir, a.images, a.onnx) diff --git a/tools/vision_qdq.py b/tools/vision_qdq.py index 069b37833..232864762 100644 --- a/tools/vision_qdq.py +++ b/tools/vision_qdq.py @@ -13,8 +13,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# Copyright 2026 bong-water-water-bong -# SPDX-License-Identifier: Apache-2.0 """Streaming QDQ A8W8 for the vision graph: one weight tensor in memory at a time. Used by `export_qwen3vl_vision.py quantize --stream`. @@ -23,11 +21,13 @@ the OOM killer), which a shared box cannot spare. This does the same job by editing the graph directly: * weights: per-output-channel symmetric int8, emitted as QuantizeLinear -> DequantizeLinear pairs; - * activations: per-tensor symmetric int8 before each MatMul, with the scale calibrated from a few images (the fp32 - graph is run once with every MatMul input exposed as an output, so no intermediate probing library is needed); + * activations: per-tensor symmetric int8 in front of every `Gemm` (the projections; attention's activation-times- + activation `MatMul`s are left float, see `matmul_weights`), with the scale calibrated from a few images -- the fp32 + graph is run once with every `Gemm` input exposed as an output, so no intermediate probing library is needed; * everything else (Conv, norms, softmax, split) stays float. - qdq_stream.py [--calib N] + vision_qdq.py [--calib N] (normally reached as `export_qwen3vl_vision.py + quantize --stream`) """ import argparse import json @@ -38,8 +38,6 @@ import onnx from onnx import TensorProto, helper, numpy_helper -sys.path.insert(0, "/tmp/qvl") - def matmul_weights(model): """(node, initializer, per_output_axis) for every layer that multiplies by a WEIGHT initializer. @@ -75,32 +73,30 @@ def expose_outputs(model, names): model.graph.output.append(helper.make_tensor_value_info(nm, TensorProto.FLOAT, None)) -def run_calibration(path, names, images, bucket, patch, temporal, real_image="", calib_pv="", clip_pct=99.9): +def make_image(rng, gh, gw, patch, temporal): + """A deterministic image at the exported bucket: two gradients plus a chequerboard, in [0, 1] before normalising.""" + H, W = gh * patch, gw * patch + yy, xx = np.mgrid[0:H, 0:W] + img = np.zeros((3, H, W), dtype=np.float32) + img[0] = (yy / max(H - 1, 1)) * 0.8 + 0.1 + img[1] = (xx / max(W - 1, 1)) * 0.8 + 0.1 + img[2] = ((yy // 32 + xx // 32) % 2) * 0.7 + 0.1 + img += rng.normal(0, 0.02, img.shape).astype(np.float32) + img = np.clip(img, 0, 1) + blocks = img.reshape(3, gh, patch, gw, patch).transpose(1, 3, 0, 2, 4) + if temporal != 1: + blocks = np.repeat(blocks[:, :, None], temporal, axis=2) + return blocks.reshape(gh * gw, 3 * temporal * patch * patch).astype(np.float32) + + +def run_calibration(path, names, images, bucket, patch, temporal, calib_pv="", clip_pct=99.9): import onnxruntime as ort - # the same deterministic image generator the tool's check rows use, copied here so the streaming quantiser does not - # depend on the harness script - from hybrid_input import image_rows # noqa: F401 (the real image path is exercised by the check rows) - def make_image(rng, gh, gw, patch, temporal): - H, W = gh * patch, gw * patch - yy, xx = np.mgrid[0:H, 0:W] - img = np.zeros((3, H, W), dtype=np.float32) - img[0] = (yy / max(H - 1, 1)) * 0.8 + 0.1 - img[1] = (xx / max(W - 1, 1)) * 0.8 + 0.1 - img[2] = ((yy // 32 + xx // 32) % 2) * 0.7 + 0.1 - img += rng.normal(0, 0.02, img.shape).astype(np.float32) - img = np.clip(img, 0, 1) - blocks = img.reshape(3, gh, patch, gw, patch).transpose(1, 3, 0, 2, 4) - if temporal != 1: - blocks = np.repeat(blocks[:, :, None], temporal, axis=2) - return blocks.reshape(gh * gw, 3 * temporal * patch * patch).astype(np.float32) sess = ort.InferenceSession(path, providers=["CPUExecutionProvider"]) got = {nm: 0.0 for nm in names} # the max over images of each image's p99.9 |x| (see --clip-percentile) feeds = [] if calib_pv: # the real picture first: synthetic gradients do not bound a photo feeds.append(np.load(calib_pv).astype(np.float32)) - elif real_image: - feeds.append(image_rows(real_image)) rng = np.random.default_rng(3) for _ in range(images): feeds.append(make_image(rng, bucket, bucket, patch, temporal)) @@ -116,10 +112,11 @@ def make_image(rng, gh, gw, patch, temporal): return got -def quantise(src, dst, calib=6, clip_percentile=99.9, calib_pv="", bucket=28, patch=16, temporal=2): +def quantise(src, dst, calib=6, clip_percentile=99.9, calib_pv="", bucket=28, patch=16, temporal=2, + bar=0.999): """Quantise `src` into `dst` (+ `dst`.data). Returns the number of quantised weights.""" a = argparse.Namespace(src=src, dst=dst, calib=calib, bucket=bucket, patch=patch, temporal=temporal, - no_act_qdq=False, real_image="", calib_pv=calib_pv, clip_percentile=clip_percentile) + no_act_qdq=False, calib_pv=calib_pv, clip_percentile=clip_percentile, bar=bar) return _run(a) @@ -132,8 +129,8 @@ def main(): ap.add_argument("--bucket", type=int, default=28) ap.add_argument("--patch", type=int, default=16) ap.add_argument("--temporal", type=int, default=2) - ap.add_argument("--real-image", default="", help="calibrate on this image too (needs PIL)") ap.add_argument("--calib-pv", default="", help="calibrate on these pixels (a .npy), so no PIL is needed here") + ap.add_argument("--bar", type=float, default=0.999, help="self-check cosine bar against the fp32 graph") ap.add_argument("--clip-percentile", type=float, default=99.9, help="per-tensor |x| percentile used as the activation range (100 = plain max, which loses a tensor " "to one outlier: measured 103/104 tensors diverge)") @@ -141,6 +138,7 @@ def main(): def _run(a): + os.makedirs(os.path.dirname(os.path.abspath(a.dst)), exist_ok=True) model = onnx.load(a.src) # ~2 GB in memory; the weights are not copied per tensor pairs, inits = matmul_weights(model) print(f"{len(pairs)} MatMul weights to quantise") @@ -152,8 +150,7 @@ def _run(a): cpath = a.dst + ".calib.onnx" onnx.save(calib_model, cpath) del calib_model - act_absmax = run_calibration(cpath, names, a.calib, a.bucket, a.patch, a.temporal, a.real_image, a.calib_pv, - a.clip_percentile) + act_absmax = run_calibration(cpath, names, a.calib, a.bucket, a.patch, a.temporal, a.calib_pv, a.clip_percentile) os.unlink(cpath) vals = sorted(act_absmax.values()) tiny = [nm for nm, v in act_absmax.items() if v < 1e-3] @@ -229,7 +226,31 @@ def _run(a): location=os.path.basename(a.dst) + ".data", size_threshold=1024) print(f"wrote {a.dst} ({os.path.getsize(a.dst) / 1e6:.1f} MB) + .data " f"({os.path.getsize(a.dst + '.data') / 1e6:.1f} MB); {len(replaced)} weights, {len(act_emitted)} activations") - return len(replaced) + return self_check(a.src, a.dst, images=3, bar=a.bar) + + +def self_check(fp32_path, qdq_path, images=3, bar=0.999): + """Cosine of every output against the fp32 graph on the same images; 1 if it clears `bar`, 0 if it does not. + + The ORT path in export_qwen3vl_vision.py gates itself the same way, and the point is that a graph below the bar must + not exit 0: the A8W8 output measures 0.61-0.91, so this path has to be able to fail. + """ + import onnxruntime as ort + + a = ort.InferenceSession(fp32_path, providers=["CPUExecutionProvider"]) + b = ort.InferenceSession(qdq_path, providers=["CPUExecutionProvider"]) + rng = np.random.default_rng(11) + worst = 1.0 + for _ in range(images): + pv = make_image(rng, 28, 28, 16, 2) + feed = {"pixel_values": pv, "image_grid_thw": np.array([[1, 28, 28]], dtype=np.int64)} + for x, y in zip(a.run(None, feed), b.run(None, feed)): + x, y = x.reshape(-1).astype(np.float64), y.reshape(-1).astype(np.float64) + worst = min(worst, float(np.dot(x, y) / (np.linalg.norm(x) * np.linalg.norm(y) + 1e-12))) + ok = worst >= bar + print(f"self-check: worst cosine {worst:.6f} against the fp32 graph on {images} images " + f"({'PASS' if ok else 'FAIL'}; the bar is {bar})") + return 0 if ok else 1 if __name__ == "__main__": From 1e1df277473befe08b09d8e9ec070a342a7b005b Mon Sep 17 00:00:00 2001 From: pi-npufeat Date: Sun, 8 Nov 2026 17:22:32 -0400 Subject: [PATCH 09/10] npu: the hybrid probe takes --top-k, because two decoders that agree on a token are not the same decoder Gate 2's remaining question is where the image prompt goes wrong, and tokens cannot answer it: two decoders can pick the same argmax and be far apart underneath, or diverge on a coin flip. --top-k N prints the top-N logits wherever an argmax is taken (prefill, and the first decode step by name), so a step can be diffed against the reference number for number. It did its job on the first run. Against HF on the same ids, positions and pooled features: prefill ours: 32:35.67 1986:33.47 785:29.32 2082:28.01 HF: 32:33.00 2082:31.25 1986:30.25 785:28.50 step 1 ours: 23:21.61 16:21.36 11:20.76 15:20.53 HF: 16771:27.13 12955:26.00 23790:24.50 36190:24.00 text-only, step 1 ours: 1697:22.63 3265:22.07 ... HF: 3265:23.75 1697:23.38 ... So the prefill agrees with the reference (same four tokens, same order bar one swap, values within 2.7) and the text-only decode agrees too (same two candidates, swapped, within 1.7) -- while the image prompt's first decode step is flat at ~21 where the reference is peaked at 27. That says the vision path, the position planes, the deepstack sets and the prefill KV are right, and the defect is in the single-token step once the cache holds image rows. --- npu/forward/hybrid_prompt_probe.cpp | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/npu/forward/hybrid_prompt_probe.cpp b/npu/forward/hybrid_prompt_probe.cpp index 7b0600857..95a0aab65 100644 --- a/npu/forward/hybrid_prompt_probe.cpp +++ b/npu/forward/hybrid_prompt_probe.cpp @@ -32,6 +32,8 @@ #include #include #include +#include +#include #include template @@ -51,9 +53,10 @@ int main(int argc, char** argv) { std::fprintf(stderr, "usage: %s [--decode N]\n", argv[0]); return 2; } - int decode = 8; + int decode = 8, top_k_n = 0; for (int i = 4; i + 1 < argc; i++) if (!std::strcmp(argv[i], "--decode")) decode = std::atoi(argv[i + 1]); + else if (!std::strcmp(argv[i], "--top-k") && i + 1 < argc) top_k_n = std::atoi(argv[++i]); const std::string dir = argv[3]; Q4nxNpuForward f; @@ -131,11 +134,29 @@ int main(int argc, char** argv) { if (v[i] > v[(size_t)b]) b = (int)i; return b; }; + // --top-k N prints the top-N logits wherever an argmax is taken, so a decode step can be diffed against the + // reference number for number instead of token for token (two decoders that agree on the argmax can still differ by + // a lot underneath, and that is what decides whether the next step is off). + auto top_k = [&](const char* where) { + if (top_k_n <= 0) return; + std::vector> v; + v.reserve(logits.size()); + for (size_t i = 0; i < logits.size(); i++) v.emplace_back(logits[i], (int)i); + const size_t k = std::min((size_t)top_k_n, v.size()); + std::partial_sort(v.begin(), v.begin() + (long)k, v.end(), + [](const std::pair& a, const std::pair& b) { return a.first > b.first; }); + std::printf("top%d %s:", top_k_n, where); + for (size_t i = 0; i < k; i++) std::printf(" %d:%.4f", v[i].second, v[i].first); + std::printf("\n"); + std::fflush(stdout); + }; std::vector out; + top_k("prefill"); out.push_back(argmax(logits)); int pos = int(positions[0 * rows + rows - 1]) + 1; // the generation step's position (delta already applied) for (int i = 1; i < decode; i++) { if (!f.step(out.back(), pos + i - 1, logits)) { std::fprintf(stderr, "step: %s\n", f.last_error()); return 1; } + top_k(i == 1 ? "step1" : "step"); out.push_back(argmax(logits)); } std::printf("prompt %d tokens (%zu image rows), deepstack %d set(s)\n", rows, rows_img, in.deepstack_sets); From 6a5eb0e712cf6ea666e49cfd7adef9df7d572485 Mon Sep 17 00:00:00 2001 From: pi-npufeat Date: Sun, 8 Nov 2026 17:47:06 -0400 Subject: [PATCH 10/10] npu: the decode step takes the cache slot separately from the rotary position The image prompt now captions. "A pixelated digital art piece featuring a" against the reference's "A retro-style video game screen showing a" -- two readings of the same picture, where every previous head stopped at "A\n<|im_end|>". The review found the real reason, and it was mine: `step()` used one number for two jobs. `pos` rotated the query (a MRoPE position, with the rope delta applied) and also set the attention window (`T = pos + 1`), while the new K/V is appended at the END of the cache. For a text prompt those coincide, which is why the text-only control was always coherent. After an image they do not: the 196 image rows walk a small h/w grid, so the position after the image is 36 while the cache holds 220 rows. The first generated token therefore attended to rows 0..36 only -- the prompt with the question cut off -- and every later token appended past the window it was reading, so it never saw what it had written. `step(token, pos, logits, cache_pos = -1)` now takes the slot explicitly and `attention()` gets it as the existing `kv_len` (its only effect is `T = kv_len > 0 ? kv_len : pos + 1`, so the rotary side is untouched). The probe passes `cache_pos = rows + i - 1`, the index the token lands in. Text prompts pass -1 and are unchanged, and the control proves it: `32 1697 594 3579 374 25244 95701 553` before and after. Top-5 on the same inputs, after the fix: step 1 ours: 12955:27.90 7377:26.27 6366:26.02 16771:25.56 HF: 16771:27.13 12955:26.00 23790:24.50 36190:24.00 Same candidate set, the top two swapped inside a 2.3 gap -- an int8/q4nx-divergence-shaped difference, not a structural one. Prefill is unchanged and still matches the reference (32, 1986, 785, 2082; values within 2.7). Also from the same review: * **DeepStack on a MoE tail.** The add sat at the end of the layer body, which a MoE layer never reaches (`continue`). It is now a lambda called on both tails, so a Qwen3-VL-*-A3B MoE layer gets its set too. Latent for this dense pack. * `q4nx_semantics.h` said a contiguous split misplaces "40 of the 64"; the measured number, in the test and the commit, is **43**. * `self_check`'s docstring had its return value inverted; the code was right. * `cur_layer_ = l`'s comment described the history of my own edit; it says what the variable is for now. q4nx_mrope_test, npu_rmsnorm_offset_test and q4nx_layer_kinds_test all pass. --- npu/forward/hybrid_prompt_probe.cpp | 12 ++++++-- npu/forward/include/q4nx_forward.h | 7 ++++- npu/forward/include/q4nx_semantics.h | 4 +-- npu/forward/src/q4nx_forward.cpp | 41 ++++++++++++++++------------ tools/vision_qdq.py | 2 +- 5 files changed, 42 insertions(+), 24 deletions(-) diff --git a/npu/forward/hybrid_prompt_probe.cpp b/npu/forward/hybrid_prompt_probe.cpp index 95a0aab65..5f1a92cba 100644 --- a/npu/forward/hybrid_prompt_probe.cpp +++ b/npu/forward/hybrid_prompt_probe.cpp @@ -153,9 +153,17 @@ int main(int argc, char** argv) { std::vector out; top_k("prefill"); out.push_back(argmax(logits)); - int pos = int(positions[0 * rows + rows - 1]) + 1; // the generation step's position (delta already applied) + int pos = int(positions[0 * rows + rows - 1]) + 1; // the generation step's ROTARY position (delta applied) + // ...and the slot the token lands in, which is the prompt's length, not that position. They coincide for a text + // prompt and do not after an image: the image rows walk a small h/w grid, so `pos` is far below the cache length and + // a step that used it as the window read only the first `pos + 1` rows -- losing the question and never attending to + // what it had just generated (the review of engine #1498 caught this; it is why every image answer was bad). for (int i = 1; i < decode; i++) { - if (!f.step(out.back(), pos + i - 1, logits)) { std::fprintf(stderr, "step: %s\n", f.last_error()); return 1; } + const int cache_pos = rows + i - 1; + if (!f.step(out.back(), pos + i - 1, logits, cache_pos)) { + std::fprintf(stderr, "step: %s\n", f.last_error()); + return 1; + } top_k(i == 1 ? "step1" : "step"); out.push_back(argmax(logits)); } diff --git a/npu/forward/include/q4nx_forward.h b/npu/forward/include/q4nx_forward.h index 2b2ef0e2a..16421ce53 100644 --- a/npu/forward/include/q4nx_forward.h +++ b/npu/forward/include/q4nx_forward.h @@ -127,7 +127,12 @@ class Q4nxNpuForward { bool extend_context(int n_tokens); // Run one token at absolute position pos; fills logits (size NV). - bool step(int token, int pos, std::vector& logits); + // `pos` is the ROTARY position; `cache_pos` is the slot the new K/V lands in, which is the only thing that + // decides how many cached rows attention reads. They are the same number only while positions equal indices: + // after an image they are not (an image's rows walk a small h/w grid, so the MRoPE position is far below the + // cache length), and using the rotary position as the window dropped the prompt's tail -- the question -- and + // every token after the first never saw what it had generated. -1 keeps the old behaviour for text prompts. + bool step(int token, int pos, std::vector& logits, int cache_pos = -1); // Batched prefill: run the whole prompt in one pass so each projection's // weight stream is read once per batch instead of once per token. The kernels diff --git a/npu/forward/include/q4nx_semantics.h b/npu/forward/include/q4nx_semantics.h index 0d43764b5..e29f53e28 100644 --- a/npu/forward/include/q4nx_semantics.h +++ b/npu/forward/include/q4nx_semantics.h @@ -215,8 +215,8 @@ int q4nx_semantics_from_config(const char* json_path, const char* model_dir, * = 2 if i % 3 == 2 && i < 3 * section[2] * = 0 otherwise * - * For [24, 20, 20] that is 24 t / 20 h / 20 w frequencies, 40 of the 64 in positions a contiguous split would not put - * them in -- which is why treating the sections as contiguous runs left every image prompt's answer bad while all + * For [24, 20, 20] that is 24 t / 20 h / 20 w frequencies, and 43 of the 64 sit where a contiguous split would not put + * them -- which is why treating the sections as contiguous runs left every image prompt's answer bad while all * equal-plane (text-only) prompts stayed exact. Frequency index is the index within head_dim/2. */ static inline int q4nx_mrope_axis(int freq, int n_section, const int* section) { diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 001e73f8d..9c2f6eb15 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -3234,9 +3234,22 @@ bool Q4nxNpuForward::step_batch(const std::vector& tokens, int start_pos, embed_scale_; if (phb) { g_ph.embed = ph_ms(tp_bs); tp_lap = std::chrono::steady_clock::now(); } + // Deepstack, in the reference's place (transformers Qwen3VLTextModel.forward: `_deepstack_process` is called + // after `decoder_layer(...)`, for layer_idx < len(sets)). Set `l` of `deepstack_` goes to the image rows of the + // residual stream after text layer `l` runs. It is a lambda because a MoE layer ends in `continue`, which used to + // skip the add entirely -- latent for this dense pack, wrong for the Qwen3-VL-*-A3B MoE models. + auto deepstack_add = [&](int l) { + if (!deepstack_ || l < 0 || l >= sem_.n_deepstack) return; + const float* add = deepstack_ + size_t(l) * size_t(deepstack_rows_) * size_t(H); + const int lo = std::max(0, deepstack_lo_), hi = std::min((int)M, deepstack_hi_); + for (int r = lo; r < hi; r++) + for (size_t k = 0; k < (size_t)H; k++) + X[size_t(r) * (size_t)H + k] += add[size_t(r - lo) * (size_t)H + k]; + }; + std::vector QKV, CTX, O, ACT, HH, D, qkv_row, ctx_row; for (int l = 0; l < c_.NL; l++) { - cur_layer_ = l; // (removed with the old pre-layer deepstack block by accident; every layer read the last one) + cur_layer_ = l; // the layer index the per-layer helpers read const bool cached = weight_cached(0); const int lt0 = (l < sem_.n_layer_types) ? sem_.layer_types[l] : Q4NX_LAYER_FULL; // A GDN layer's host parameters (conv window, alpha/beta, ssm_a/dt, gated @@ -3448,6 +3461,7 @@ bool Q4nxNpuForward::step_batch(const std::vector& tokens, int start_pos, for (int k = 0; k < H; k++) X[(size_t)m * H + k] += d_row[(size_t)k] * residual_scale_; } + deepstack_add(l); // a MoE layer's tail still owes the layer its set continue; } // ACT is read below at the *fused* design's N, so both branches have to hand it rows at that @@ -3537,21 +3551,7 @@ bool Q4nxNpuForward::step_batch(const std::vector& tokens, int start_pos, fprintf(stderr, " [LB] m=%d l=%d %.6f %.4f\n", m, l, ms / H, mmx); } } - // Deepstack, in the reference's place: set `l` is added to the hidden state AFTER text decoder layer `l` - // runs, for l < number of sets (transformers Qwen3VLTextModel.forward: "add visual features to the hidden - // states of first several layers", `hidden_states = _deepstack_process(hidden_states, mask, sets[layer_idx])` - // inside the loop, after the call). The pack's `deepstack_visual_indexes` are the VISION tower's layers the - // sets are tapped from (5/11/17 for this model) and are not text layer indexes -- using them here put the - // three sets at text layers 8/16/24, before the layer, which is why the image prompts answered "A\n". - if (deepstack_ && l < sem_.n_deepstack) { - // Only the image rows: a set has rows_img rows, and the reference adds it at the image-token positions, - // not across the prompt. - const float* add = deepstack_ + size_t(l) * size_t(deepstack_rows_) * size_t(H); - const int lo = std::max(0, deepstack_lo_), hi = std::min((int)M, deepstack_hi_); - for (int r = lo; r < hi; r++) - for (size_t k = 0; k < (size_t)H; k++) - X[size_t(r) * (size_t)H + k] += add[size_t(r - lo) * (size_t)H + k]; - } + deepstack_add(l); } if (phb) { g_ph.layer = ph_ms(tp_lap); tp_lap = std::chrono::steady_clock::now(); } @@ -4257,7 +4257,7 @@ bool Q4nxNpuForward::mlp(const float* gu, float* h, int im) { // -------------------------------------------------------------------------- // step // -------------------------------------------------------------------------- -bool Q4nxNpuForward::step(int token, int pos, std::vector& logits) { +bool Q4nxNpuForward::step(int token, int pos, std::vector& logits, int cache_pos) { if (!ready_) { err_ = "not initialised"; return false; } // Generation is causal by construction (one position at a time, keys only in the past), so a pack // whose full-attention layers read the whole prompt cannot be run this way. Refused by name @@ -4482,8 +4482,13 @@ bool Q4nxNpuForward::step(int token, int pos, std::vector& logits) { } { LhScope _tlh(LH_ATTN); + // The rotary position rotates q; the cache slot decides the window. Passing the rotary position as the + // window is what broke every image prompt: an image's rows walk a small h/w grid, so the position after the + // image is far below the cache length, and attention then read only the first `pos + 1` rows -- dropping + // the prompt's tail (the question) and never seeing the tokens it had just generated. + const int kv_window = cache_pos >= 0 ? cache_pos + 1 : 0; if (!attention(qkv_use, pos, nullptr, ctx, l_hd, l_nkv_heads, l_theta, l_rot, l_prop, - l_win, kv_layer, !kv_shared)) { + l_win, kv_layer, !kv_shared, kv_window)) { err_ = "attention failed"; return false; } diff --git a/tools/vision_qdq.py b/tools/vision_qdq.py index 232864762..15194cbf9 100644 --- a/tools/vision_qdq.py +++ b/tools/vision_qdq.py @@ -230,7 +230,7 @@ def _run(a): def self_check(fp32_path, qdq_path, images=3, bar=0.999): - """Cosine of every output against the fp32 graph on the same images; 1 if it clears `bar`, 0 if it does not. + """Cosine of every output against the fp32 graph on the same images; returns 0 if it clears `bar`, 1 if not. The ORT path in export_qwen3vl_vision.py gates itself the same way, and the point is that a graph below the bar must not exit 0: the A8W8 output measures 0.61-0.91, so this path has to be able to fail.