diff --git a/CMakeLists.txt b/CMakeLists.txt index 26e263985..5fe62637f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1267,6 +1267,18 @@ if(ONEBIT_NPU) if(ONEBIT_NPU) # The decoder's precomputed-embedding entry point (npu/forward/embeds_equiv_probe.cpp): rows built from the # model's own table must give byte-identical logits to the token-id path, on the host and on the array. + # The three-plane (MRoPE) position input: equal planes must be byte-identical to plain RoPE, different planes + # must move the logits (npu/forward/mrope_planes_probe.cpp). + # The wiring a vision-conditioned prompt needs on the engine side: vision rows + 3-plane positions + deepstack + # sets through step_batch_embeds(), then a greedy decode (npu/forward/hybrid_prompt_probe.cpp). + add_executable(hybrid_prompt_probe npu/forward/hybrid_prompt_probe.cpp) + target_include_directories(hybrid_prompt_probe PRIVATE ${CMAKE_SOURCE_DIR}/npu + ${CMAKE_SOURCE_DIR}/npu/forward/include ${CMAKE_SOURCE_DIR}/npu/forward/engine-src) + target_link_libraries(hybrid_prompt_probe PRIVATE onebit_npu_forward onebit_npu_rt) + add_executable(mrope_planes_probe npu/forward/mrope_planes_probe.cpp) + target_include_directories(mrope_planes_probe PRIVATE ${CMAKE_SOURCE_DIR}/npu + ${CMAKE_SOURCE_DIR}/npu/forward/include ${CMAKE_SOURCE_DIR}/npu/forward/engine-src) + target_link_libraries(mrope_planes_probe PRIVATE onebit_npu_forward onebit_npu_rt) add_executable(embeds_equiv_probe npu/forward/embeds_equiv_probe.cpp) target_include_directories(embeds_equiv_probe PRIVATE ${CMAKE_SOURCE_DIR}/npu ${CMAKE_SOURCE_DIR}/npu/forward/include ${CMAKE_SOURCE_DIR}/npu/forward/engine-src) diff --git a/npu/forward/hybrid_prompt_probe.cpp b/npu/forward/hybrid_prompt_probe.cpp new file mode 100644 index 000000000..5f1a92cba --- /dev/null +++ b/npu/forward/hybrid_prompt_probe.cpp @@ -0,0 +1,175 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// A vision-conditioned prompt through the decoder (engine#1311/TQVL): the wiring gate 2 needs, on the engine side. +// +// Reads what the vision half produced -- the prompt's token ids, the image rows' embeddings, the 3-plane positions and +// the deepstack sets -- and runs the prefill through step_batch_embeds(), then greedily decodes N tokens. Text rows' +// embeddings come from the model's own table (embed_row), so the only foreign rows are the image ones. +// +// hybrid_prompt_probe [--decode N] +// +// holds ids.i32, positions.i32 (3 x rows), pooled.f32 (rows_img x H), deepstack.f32 (sets x rows_img x H) +// and meta.json {"image_lo": i, "image_hi": j}. Build them with the vision graph (tools/export_qwen3vl_vision.py) +// plus the tokenizer; this probe does not touch a tokenizer. +#include "q4nx_forward.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +template +static bool read_vec(const std::string& path, std::vector& out) { + std::ifstream f(path, std::ios::binary); + if (!f) return false; + f.seekg(0, std::ios::end); + const auto n = (size_t)f.tellg() / sizeof(T); + f.seekg(0); + out.resize(n); + f.read((char*)out.data(), (std::streamsize)(n * sizeof(T))); + return (bool)f; +} + +int main(int argc, char** argv) { + if (argc < 4) { + std::fprintf(stderr, "usage: %s [--decode N]\n", argv[0]); + return 2; + } + int decode = 8, top_k_n = 0; + for (int i = 4; i + 1 < argc; i++) + if (!std::strcmp(argv[i], "--decode")) decode = std::atoi(argv[i + 1]); + else if (!std::strcmp(argv[i], "--top-k") && i + 1 < argc) top_k_n = std::atoi(argv[++i]); + const std::string dir = argv[3]; + + Q4nxNpuForward f; + std::unique_ptr dev; + bool ok; + if (getenv("NPU_INFER_HOST_GEMM")) { + ok = f.init_host(argv[1], argv[2]); + } else { + dev = std::make_unique(0); + ok = f.init(*dev, argv[1], argv[2]); + } + if (!ok) { std::fprintf(stderr, "init: %s\n", f.last_error()); return 1; } + const int H = 0; // read from a row, below + (void)H; + + std::vector ids, positions; + std::vector pooled, deepstack; + if (!read_vec(dir + "/ids.i32", ids) || !read_vec(dir + "/positions.i32", positions) || + !read_vec(dir + "/pooled.f32", pooled) || !read_vec(dir + "/deepstack.f32", deepstack)) { + std::fprintf(stderr, "cannot read the input files from %s\n", dir.c_str()); + return 2; + } + // meta.json is tiny; the two numbers are read by position so the probe needs no JSON library + int meta[2] = {-1, -1}; + { + std::ifstream m(dir + "/meta.txt"); + if (m) m >> meta[0] >> meta[1]; // image_lo image_hi (one line each is fine) + } + // image_lo == image_hi is the text-only control: no image rows, nothing to inject. + if (meta[0] < 0 || meta[1] < meta[0]) { std::fprintf(stderr, "bad meta (image_lo image_hi)\n"); return 2; } + const int rows = (int)ids.size(); + std::vector one; + f.embed_row(11, one); + const size_t Hd = one.size(); + const size_t rows_img = size_t(meta[1] - meta[0]); + if (pooled.size() != rows_img * Hd || positions.size() != size_t(3) * size_t(rows)) { + std::fprintf(stderr, "shapes: pooled %zu (want %zu), positions %zu (want %zu)\n", pooled.size() / Hd, + rows_img, positions.size(), size_t(3) * rows); + return 2; + } + const int n_ds = f.n_deepstack_layers(); + if (n_ds > 0 && deepstack.size() != size_t(n_ds) * rows_img * Hd) { + std::fprintf(stderr, "deepstack %zu, want %d sets x %zu rows x %zu\n", deepstack.size(), n_ds, rows_img, Hd); + return 2; + } + + // embedding rows: the model's own for text positions, the vision graph's for the image span + std::vector embeds; + embeds.reserve(size_t(rows) * Hd); + for (int r = 0; r < rows; r++) { + if (r >= meta[0] && r < meta[1]) { + const float* src = pooled.data() + size_t(r - meta[0]) * Hd; + embeds.insert(embeds.end(), src, src + Hd); + } else { + f.embed_row(ids[size_t(r)], one); + embeds.insert(embeds.end(), one.begin(), one.end()); + } + } + Q4nxNpuForward::HybridInputs in; + in.position_ids = &positions; + // A text-only prompt (the control) has no image rows, so there is no span to add deepstack sets at. + if (n_ds > 0 && rows_img > 0) { + in.deepstack = deepstack.data(); + in.deepstack_sets = n_ds; + in.image_lo = meta[0]; + in.image_hi = meta[1]; + } + + std::vector logits; + f.reset(); + if (!f.step_batch_embeds(embeds, 0, logits, in)) { std::fprintf(stderr, "prefill: %s\n", f.last_error()); return 1; } + auto argmax = [](const std::vector& v) { + int b = 0; + for (size_t i = 1; i < v.size(); i++) + if (v[i] > v[(size_t)b]) b = (int)i; + return b; + }; + // --top-k N prints the top-N logits wherever an argmax is taken, so a decode step can be diffed against the + // reference number for number instead of token for token (two decoders that agree on the argmax can still differ by + // a lot underneath, and that is what decides whether the next step is off). + auto top_k = [&](const char* where) { + if (top_k_n <= 0) return; + std::vector> v; + v.reserve(logits.size()); + for (size_t i = 0; i < logits.size(); i++) v.emplace_back(logits[i], (int)i); + const size_t k = std::min((size_t)top_k_n, v.size()); + std::partial_sort(v.begin(), v.begin() + (long)k, v.end(), + [](const std::pair& a, const std::pair& b) { return a.first > b.first; }); + std::printf("top%d %s:", top_k_n, where); + for (size_t i = 0; i < k; i++) std::printf(" %d:%.4f", v[i].second, v[i].first); + std::printf("\n"); + std::fflush(stdout); + }; + std::vector out; + top_k("prefill"); + out.push_back(argmax(logits)); + int pos = int(positions[0 * rows + rows - 1]) + 1; // the generation step's ROTARY position (delta applied) + // ...and the slot the token lands in, which is the prompt's length, not that position. They coincide for a text + // prompt and do not after an image: the image rows walk a small h/w grid, so `pos` is far below the cache length and + // a step that used it as the window read only the first `pos + 1` rows -- losing the question and never attending to + // what it had just generated (the review of engine #1498 caught this; it is why every image answer was bad). + for (int i = 1; i < decode; i++) { + const int cache_pos = rows + i - 1; + if (!f.step(out.back(), pos + i - 1, logits, cache_pos)) { + std::fprintf(stderr, "step: %s\n", f.last_error()); + return 1; + } + top_k(i == 1 ? "step1" : "step"); + out.push_back(argmax(logits)); + } + std::printf("prompt %d tokens (%zu image rows), deepstack %d set(s)\n", rows, rows_img, in.deepstack_sets); + std::printf("generated %d token ids:", (int)out.size()); + for (int t : out) std::printf(" %d", t); + std::printf("\n"); + return 0; +} diff --git a/npu/forward/include/q4nx_forward.h b/npu/forward/include/q4nx_forward.h index e1fad2d3d..16421ce53 100644 --- a/npu/forward/include/q4nx_forward.h +++ b/npu/forward/include/q4nx_forward.h @@ -127,7 +127,12 @@ class Q4nxNpuForward { bool extend_context(int n_tokens); // Run one token at absolute position pos; fills logits (size NV). - bool step(int token, int pos, std::vector& logits); + // `pos` is the ROTARY position; `cache_pos` is the slot the new K/V lands in, which is the only thing that + // decides how many cached rows attention reads. They are the same number only while positions equal indices: + // after an image they are not (an image's rows walk a small h/w grid, so the MRoPE position is far below the + // cache length), and using the rotary position as the window dropped the prompt's tail -- the question -- and + // every token after the first never saw what it had generated. -1 keeps the old behaviour for text prompts. + bool step(int token, int pos, std::vector& logits, int cache_pos = -1); // Batched prefill: run the whole prompt in one pass so each projection's // weight stream is read once per batch instead of once per token. The kernels @@ -148,6 +153,29 @@ class Q4nxNpuForward { // The per-token fallback path (MLA/MoE/GDN/... models) needs ids and is refused by name. bool step_batch_embeds(const std::vector& embeds, int start_pos, std::vector& logits); + // What a vision-conditioned prompt adds on top of the embeddings (engine#1311/TQVL; plan P4's `rope.planes` + // and a per-layer feature injection): + // position_ids 3 x rows, plane-major (t, h, w) — the layout the reference's `get_rope_index` produces and + // AMD's `text.onnx` takes. Text rows have all three equal, which is bit-identical to plain RoPE + // (measured: 0 difference over positions 0..262143; image rows differ per axis and that is the + // whole point). A pack that declares mrope without sections is refused by name. + // deepstack sets x rows x H, one set per `vision_config.deepstack_visual_indexes` entry (Qwen3-VL: 5/11/17); + // set s is added to the hidden state when the layer loop reaches deepstack_layers[s]. This is how + // the reference injects the vision tower's extra features, and an answer cannot match it without. + struct HybridInputs { + const std::vector* position_ids = nullptr; + const float* deepstack = nullptr; + int deepstack_sets = 0; + // The rows the deepstack sets belong to: the image span. They are added THERE, not everywhere -- a set has + // rows_img rows, so adding it to every row of the prompt would both be the wrong function and read past the + // buffer (the first version of this did exactly that and produced garbage tokens). + int image_lo = -1; + int image_hi = -1; + }; + const int* deepstack_layers() const { return sem_.deepstack_layers; } + int n_deepstack_layers() const { return sem_.n_deepstack; } + bool step_batch_embeds(const std::vector& embeds, int start_pos, std::vector& logits, + const HybridInputs& in); // The same batched pass, but returning the logits of EVERY row (M x NV, row-major) instead of only // the last one. Speculative decoding needs them: a draft token is accepted only if the model's own @@ -231,7 +259,7 @@ class Q4nxNpuForward { // The default for the above when NPU_FFN_CACHE says nothing: on for a GDN // model on the NPU route, with a size guard. See the definition. bool ffn_cache_default() const; - bool attention(const std::vector& qkv, int pos, std::vector& ctx, + bool attention(const std::vector& qkv, int pos, const int* pos3, std::vector& ctx, int head_dim, int n_kv_heads, float theta, int rotary_dim, bool proportional, int window, int kv_layer, bool append_kv, int kv_len = 0); @@ -354,6 +382,16 @@ class Q4nxNpuForward { std::vector embed_; // [NV*H] bf16 // Set only while step_batch_embeds() runs: the row source for the batched body's embedding fill. const float* inject_embeds_ = nullptr; + // Set only while step_batch_embeds(..., position_ids, ...) runs: 3 x rows, plane-major, or null. + const int* pos3_ = nullptr; + int pos3_rows_ = 0; + // Set only while a step_batch_embeds() call carries deepstack sets: sets x rows x H. + const float* deepstack_ = nullptr; + int deepstack_rows_ = 0; + int deepstack_lo_ = -1, deepstack_hi_ = -1; + // Which plane a rotary frequency belongs to under interleaved MRoPE. The rule lives in q4nx_semantics.h as + // q4nx_mrope_axis() so the C unit test can pin it without building a forward pass. + int mrope_plane(int freq) const { return q4nx_mrope_axis(freq, sem_.n_mrope_section, sem_.mrope_section); } // Some models store the embedding QUANTIZED (Gemma4-E2B/E4B: dtype I8), and // reading it as bf16 yields ~zero - which invalidated the whole forward and // the tied LM head (all-zero logits). When the dtype is not BF16 the diff --git a/npu/forward/include/q4nx_semantics.h b/npu/forward/include/q4nx_semantics.h index a642b262e..e29f53e28 100644 --- a/npu/forward/include/q4nx_semantics.h +++ b/npu/forward/include/q4nx_semantics.h @@ -95,6 +95,17 @@ typedef struct { /* Interleaved/multimodal RoPE variant that this forward does not implement. * Source: rope_parameters.mrope_interleaved (or rope_type != "default"). */ int mrope; + /* MVROPE sections: how many of the rotary frequencies belong to each of the (t, h, w) axes. Read from + * rope_scaling.mrope_section (Hugging Face) or rope_parameters.mrope_section (OGA/Qwen3.5); zeros and + * n_mrope_section == 0 mean "declared mrope without sections", which a consumer must refuse rather than + * guess a layout for. */ + int mrope_section[3]; + int n_mrope_section; + /* DEEPSTACK: which decoder layers take one of the vision tower's extra feature sets. Read from + * vision_config.deepstack_visual_indexes (Qwen3-VL: [5, 11, 17]); the consumer adds set s to the hidden state at + * layer deepstack_layers[s]. Empty means the pack has none. */ + int deepstack_layers[8]; + int n_deepstack; /* RoPE is configured PER LAYER TYPE (Gemma4 nests rope_parameters. * full_attention and .sliding_attention with different thetas and types). * Source: presence of rope_parameters.{full,sliding}_attention sub-blocks. */ @@ -193,6 +204,29 @@ void q4nx_semantics_default(Q4nxSemantics* s); int q4nx_semantics_from_config(const char* json_path, const char* model_dir, Q4nxSemantics* s); +/* Which axis a rotary frequency belongs to under INTERLEAVED MRoPE. + * + * The three `mrope_section` values are not contiguous runs of frequencies. The reference builds cos/sin for all three + * planes and recomposes them: start from the t plane and, for each of h and w, overwrite every third frequency with that + * plane's value over a stride-3 window of length `section[dim] * 3` (transformers + * `Qwen3VLTextRotaryEmbedding.recomposition_frequencies`: `idx = slice(offset, mrope_section[dim] * 3, 3)`). + * + * axis(i) = 1 if i % 3 == 1 && i < 3 * section[1] + * = 2 if i % 3 == 2 && i < 3 * section[2] + * = 0 otherwise + * + * For [24, 20, 20] that is 24 t / 20 h / 20 w frequencies, and 43 of the 64 sit where a contiguous split would not put + * them -- which is why treating the sections as contiguous runs left every image prompt's answer bad while all + * equal-plane (text-only) prompts stayed exact. Frequency index is the index within head_dim/2. + */ +static inline int q4nx_mrope_axis(int freq, int n_section, const int* section) { + if (n_section < 3 || !section) return 0; + const int m = freq % 3; + if (m == 1 && freq < 3 * section[1]) return 1; + if (m == 2 && freq < 3 * section[2]) return 2; + return 0; +} + /* Human-readable one-line dump of every flag with its source field. */ void q4nx_semantics_dump(const Q4nxSemantics* s); diff --git a/npu/forward/mrope_planes_probe.cpp b/npu/forward/mrope_planes_probe.cpp new file mode 100644 index 000000000..25a806411 --- /dev/null +++ b/npu/forward/mrope_planes_probe.cpp @@ -0,0 +1,109 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// MRoPE plumbing probe (engine#1311/TQVL, plan P4's `rope.planes`): does step_batch_embeds()'s three-plane position +// input do what it claims? +// +// (a) three EQUAL planes (t = h = w, i.e. a text prompt) must be BYTE-IDENTICAL to the token path -- the identity +// proved numerically (0 angle difference over positions 0..262143, theta 5e6); +// (b) three DIFFERENT planes (an image's (t, h, w) walk) must run and must move the logits, which is what an image +// prompt needs and what plain RoPE cannot express; +// (c) a pack that declares MRoPE without `mrope_section` is refused by name rather than guessed at (the parse is +// in q4nx_semantics.c; this probe runs on a pack that does carry the sections). +// +// mrope_planes_probe # device path (inside npu-hold) +// NPU_INFER_HOST_GEMM=1 mrope_planes_probe # the same body on the host, no device +#include "q4nx_forward.h" +#include +#include +#include +#include +#include + +int main(int argc, char** argv) { + if (argc < 3) { std::fprintf(stderr, "usage: %s \n", argv[0]); return 2; } + Q4nxNpuForward f; + std::unique_ptr dev; + bool ok; + if (getenv("NPU_INFER_HOST_GEMM")) { + ok = f.init_host(argv[1], argv[2]); + } else { + dev = std::make_unique(0); + ok = f.init(*dev, argv[1], argv[2]); + } + if (!ok) { std::fprintf(stderr, "init: %s\n", f.last_error()); return 1; } + const std::vector ids = {9707, 11, 1879, 374, 264, 1234, 16, 13}; + const int rows = int(ids.size()); + std::vector embeds; + for (int id : ids) { std::vector row; f.embed_row(id, row); embeds.insert(embeds.end(), row.begin(), row.end()); } + + std::vector plain, eq, diff; + f.reset(); if (!f.step_batch(ids, 0, plain)) { std::fprintf(stderr, "step_batch: %s\n", f.last_error()); return 1; } + + std::vector planes_eq(size_t(3 * rows)), planes_img(size_t(3 * rows)); + for (int r = 0; r < rows; r++) { + planes_eq[r] = planes_eq[rows + r] = planes_eq[2 * rows + r] = r; // t = h = w: text + planes_img[r] = r; // t + planes_img[rows + r] = r / 2; // h (a 2-wide image grid) + planes_img[2 * rows + r] = r % 2; // w + } + Q4nxNpuForward::HybridInputs in_eq, in_img; + in_eq.position_ids = &planes_eq; + in_img.position_ids = &planes_img; + f.reset(); if (!f.step_batch_embeds(embeds, 0, eq, in_eq)) { std::fprintf(stderr, "eq: %s\n", f.last_error()); return 1; } + f.reset(); if (!f.step_batch_embeds(embeds, 0, diff, in_img)) { std::fprintf(stderr, "img: %s\n", f.last_error()); return 1; } + + int eq_diff = 0; double img_delta = 0.0; + for (size_t i = 0; i < plain.size(); i++) { + if (plain[i] != eq[i]) eq_diff++; + img_delta += std::abs(double(plain[i]) - double(diff[i])); + } + std::printf("equal planes vs token path: %s (%d of %zu logits differ)\n", eq_diff ? "MISMATCH" : "byte-identical", + eq_diff, plain.size()); + std::printf("image-style planes vs plain : mean |dlogit| %.4f (they must differ)\n", img_delta / double(plain.size())); + const bool pass = eq_diff == 0 && img_delta > 0.0; + // Deepstack: the pack's own layer list must be read (Qwen3-VL declares three), a set count that disagrees must be + // refused by name, and non-zero sets must move the logits -- the same control/effect shape as the planes above. + const int n_ds = f.n_deepstack_layers(); + std::printf("deepstack layers declared by the pack: %d\n", n_ds); + bool ds_ok = true; + if (n_ds > 0) { + const int Hd = int(embeds.size() / size_t(rows)); + std::vector sets(size_t(n_ds) * size_t(rows) * size_t(Hd), 0.05f); + Q4nxNpuForward::HybridInputs in_ds; + in_ds.deepstack = sets.data(); + in_ds.deepstack_sets = n_ds; + std::vector with_ds; + f.reset(); + if (!f.step_batch_embeds(embeds, 0, with_ds, in_ds)) { + std::fprintf(stderr, "deepstack: %s\n", f.last_error()); + return 1; + } + double ds_delta = 0.0; + for (size_t i = 0; i < plain.size(); i++) ds_delta += std::abs(double(plain[i]) - double(with_ds[i])); + ds_delta /= double(plain.size()); + std::printf("deepstack sets vs plain : mean |dlogit| %.4f (they must differ)\n", ds_delta); + Q4nxNpuForward::HybridInputs bad; + bad.deepstack = sets.data(); + bad.deepstack_sets = n_ds + 1; + std::vector refused; + f.reset(); + const bool ran = f.step_batch_embeds(embeds, 0, refused, bad); + std::printf("wrong set count : %s (%s)\n", ran ? "RAN" : "refused by name", f.last_error()); + ds_ok = ds_delta > 0.0 && !ran; + } + const bool all = pass && ds_ok; + std::printf("%s\n", all ? "PASS" : "FAIL"); + return all ? 0 : 1; +} diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 57effaf1c..9c2f6eb15 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -3024,7 +3024,7 @@ void Q4nxNpuForward::npu_gemm_rows(int role, const float* A, int M, int K, int N // The folded container's sign table: magic "HAD1", u32 count, count x (u32 width, u32 offset), // then the f32 signs those offsets index (the GGUF's prism.hadamard.sign_widths/sign_values). bool Q4nxNpuForward::step_batch_embeds(const std::vector& embeds, int start_pos, - std::vector& logits) { + std::vector& logits, const HybridInputs& in) { if (!ready_) { err_ = "not initialised"; return false; } const size_t H = (size_t)c_.H; if (H == 0 || embeds.empty() || embeds.size() % H != 0) { @@ -3035,15 +3035,57 @@ bool Q4nxNpuForward::step_batch_embeds(const std::vector& embeds, int sta for (float v : embeds) if (!std::isfinite(v)) { err_ = "step_batch_embeds: a row holds a non-finite value"; return false; } const int rows = int(embeds.size() / H); + const std::vector& position_ids = in.position_ids ? *in.position_ids : std::vector(); + if (!position_ids.empty()) { + if (!sem_.mrope) { + err_ = "step_batch_embeds: position_ids given but this pack does not declare interleaved MRoPE"; + return false; + } + if (sem_.n_mrope_section < 3) { + err_ = "step_batch_embeds: the pack declares MRoPE but no mrope_section, so the axis layout is " + "unknown; refusing rather than guessing which frequencies are (t, h, w)"; + return false; + } + if (position_ids.size() != size_t(3) * size_t(rows)) { + err_ = "step_batch_embeds: position_ids is " + std::to_string(position_ids.size()) + " values, expected 3 x " + + std::to_string(rows); + return false; + } + } // The body indexes tokens[m] only when nothing is injected, but it still needs the row count and the guards // above it read tokens; a zero vector of the right length keeps every check honest (0 is a valid id) while // the fill itself takes the injected rows. + if (in.deepstack_sets > 0) { + if (in.deepstack_sets != sem_.n_deepstack) { + err_ = "step_batch_embeds: " + std::to_string(in.deepstack_sets) + " deepstack set(s) given but the pack " + "declares " + std::to_string(sem_.n_deepstack) + " deepstack layer(s)"; + return false; + } + if (!in.deepstack) { err_ = "step_batch_embeds: deepstack_sets without a buffer"; return false; } + if (in.image_lo < 0 || in.image_hi <= in.image_lo || in.image_hi > rows) { + err_ = "step_batch_embeds: deepstack sets need the image row span (image_lo/image_hi inside 0.." + + std::to_string(rows) + ")"; + return false; + } + } const std::vector placeholder(size_t(rows), 0); struct Restore { const float*& p; - ~Restore() { p = nullptr; } - } restore{inject_embeds_}; + const int*& q; + int& n; + const float*& d; + int& dn; + int& lo; + int& hi; + ~Restore() { p = nullptr; q = nullptr; n = 0; d = nullptr; dn = 0; lo = -1; hi = -1; } + } restore{inject_embeds_, pos3_, pos3_rows_, deepstack_, deepstack_rows_, deepstack_lo_, deepstack_hi_}; inject_embeds_ = embeds.data(); + pos3_ = position_ids.empty() ? nullptr : position_ids.data(); + pos3_rows_ = rows; + deepstack_ = in.deepstack; + deepstack_rows_ = in.image_hi - in.image_lo; + deepstack_lo_ = in.image_lo; + deepstack_hi_ = in.image_hi; return step_batch(placeholder, start_pos, logits); } @@ -3192,9 +3234,22 @@ bool Q4nxNpuForward::step_batch(const std::vector& tokens, int start_pos, embed_scale_; if (phb) { g_ph.embed = ph_ms(tp_bs); tp_lap = std::chrono::steady_clock::now(); } + // Deepstack, in the reference's place (transformers Qwen3VLTextModel.forward: `_deepstack_process` is called + // after `decoder_layer(...)`, for layer_idx < len(sets)). Set `l` of `deepstack_` goes to the image rows of the + // residual stream after text layer `l` runs. It is a lambda because a MoE layer ends in `continue`, which used to + // skip the add entirely -- latent for this dense pack, wrong for the Qwen3-VL-*-A3B MoE models. + auto deepstack_add = [&](int l) { + if (!deepstack_ || l < 0 || l >= sem_.n_deepstack) return; + const float* add = deepstack_ + size_t(l) * size_t(deepstack_rows_) * size_t(H); + const int lo = std::max(0, deepstack_lo_), hi = std::min((int)M, deepstack_hi_); + for (int r = lo; r < hi; r++) + for (size_t k = 0; k < (size_t)H; k++) + X[size_t(r) * (size_t)H + k] += add[size_t(r - lo) * (size_t)H + k]; + }; + std::vector QKV, CTX, O, ACT, HH, D, qkv_row, ctx_row; for (int l = 0; l < c_.NL; l++) { - cur_layer_ = l; + cur_layer_ = l; // the layer index the per-layer helpers read const bool cached = weight_cached(0); const int lt0 = (l < sem_.n_layer_types) ? sem_.layer_types[l] : Q4NX_LAYER_FULL; // A GDN layer's host parameters (conv window, alpha/beta, ssm_a/dt, gated @@ -3328,7 +3383,16 @@ bool Q4nxNpuForward::step_batch(const std::vector& tokens, int start_pos, in = &qkv_use; } LhScope _t_attn(LH_ATTN); - if (!attention(*in, start_pos + m, dst ? ctx_row : ctx_scratch, l_hd, l_nkv, l_theta, + // pos3_ is plane-major (3 x rows): row m's three positions are three strided reads, not a pointer + // offset, so they are gathered here into the trio attention() expects. + int trio[3] = {0, 0, 0}; + if (pos3_) { + trio[0] = pos3_[m]; + trio[1] = pos3_[pos3_rows_ + m]; + trio[2] = pos3_[2 * pos3_rows_ + m]; + } + if (!attention(*in, start_pos + m, pos3_ ? trio : nullptr, dst ? ctx_row : ctx_scratch, + l_hd, l_nkv, l_theta, l_rot, l_prop, l_win, l, append, kvlen)) { err_ = "attention failed"; return false; } @@ -3397,6 +3461,7 @@ bool Q4nxNpuForward::step_batch(const std::vector& tokens, int start_pos, for (int k = 0; k < H; k++) X[(size_t)m * H + k] += d_row[(size_t)k] * residual_scale_; } + deepstack_add(l); // a MoE layer's tail still owes the layer its set continue; } // ACT is read below at the *fused* design's N, so both branches have to hand it rows at that @@ -3486,6 +3551,7 @@ bool Q4nxNpuForward::step_batch(const std::vector& tokens, int start_pos, fprintf(stderr, " [LB] m=%d l=%d %.6f %.4f\n", m, l, ms / H, mmx); } } + deepstack_add(l); } if (phb) { g_ph.layer = ph_ms(tp_lap); tp_lap = std::chrono::steady_clock::now(); } @@ -4011,7 +4077,7 @@ bool Q4nxNpuForward::dense_mlp(int l, const std::vector& xn, std::vector< return true; } -bool Q4nxNpuForward::attention(const std::vector& qkv, int pos, +bool Q4nxNpuForward::attention(const std::vector& qkv, int pos, const int* pos3, std::vector& ctx, int head_dim, int n_kv_heads, float theta, int rotary_dim, bool proportional, int window, @@ -4088,7 +4154,10 @@ bool Q4nxNpuForward::attention(const std::vector& qkv, int pos, // nothing per token. No-op unless the config declares rope_type "llama3". inv = q4nx_rope_llama3_inv_freq(inv, &rope_llama3_); } - const double ang = (double)pos * inv; + // Interleaved MRoPE: each rotary frequency rotates by its own axis's position (t, h or w). With the + // three planes equal this is exactly `pos * inv`, which is why text-only prompts are bit-identical + // to plain RoPE (measured: 0 angle difference over positions 0..262143, engine#1311/TQVL). + const double ang = (double)(pos3 ? pos3[mrope_plane(i)] : pos) * inv; cos_[i] = (float)std::cos(ang); sin_[i] = (float)std::sin(ang); } @@ -4188,7 +4257,7 @@ bool Q4nxNpuForward::mlp(const float* gu, float* h, int im) { // -------------------------------------------------------------------------- // step // -------------------------------------------------------------------------- -bool Q4nxNpuForward::step(int token, int pos, std::vector& logits) { +bool Q4nxNpuForward::step(int token, int pos, std::vector& logits, int cache_pos) { if (!ready_) { err_ = "not initialised"; return false; } // Generation is causal by construction (one position at a time, keys only in the past), so a pack // whose full-attention layers read the whole prompt cannot be run this way. Refused by name @@ -4413,8 +4482,13 @@ bool Q4nxNpuForward::step(int token, int pos, std::vector& logits) { } { LhScope _tlh(LH_ATTN); - if (!attention(qkv_use, pos, ctx, l_hd, l_nkv_heads, l_theta, l_rot, l_prop, - l_win, kv_layer, !kv_shared)) { + // The rotary position rotates q; the cache slot decides the window. Passing the rotary position as the + // window is what broke every image prompt: an image's rows walk a small h/w grid, so the position after the + // image is far below the cache length, and attention then read only the first `pos + 1` rows -- dropping + // the prompt's tail (the question) and never seeing the tokens it had just generated. + const int kv_window = cache_pos >= 0 ? cache_pos + 1 : 0; + if (!attention(qkv_use, pos, nullptr, ctx, l_hd, l_nkv_heads, l_theta, l_rot, l_prop, + l_win, kv_layer, !kv_shared, kv_window)) { err_ = "attention failed"; return false; } diff --git a/npu/forward/src/q4nx_semantics.c b/npu/forward/src/q4nx_semantics.c index 49fd44294..bfb88d276 100644 --- a/npu/forward/src/q4nx_semantics.c +++ b/npu/forward/src/q4nx_semantics.c @@ -229,6 +229,52 @@ int q4nx_semantics_from_config(const char* json_path, const char* model_dir, * picking one of the two. */ double d = 0; double prf_rope = 0; /* partial_rotary_factor found inside rope_parameters */ + /* Deepstack layers: the vision tower emits extra feature sets that the decoder adds at these layer indexes. */ + { + const char* vc = value_for_top(json, "vision_config"); + const char* at = vc ? strstr(vc, "deepstack_visual_indexes") : NULL; + if (at) at = strchr(at, '['); + if (at) { + int got = 0; + const char* p3 = at + 1; + while (*p3 && *p3 != ']' && got < 8) { + if (*p3 >= '0' && *p3 <= '9') { + char* end = NULL; + s->deepstack_layers[got++] = (int)strtol(p3, &end, 10); + p3 = end; /* strtol already stopped on the separator; a for-clause p3++ would skip the ']' */ + } else { + p3++; + } + } + s->n_deepstack = got; + } + } + /* The sections say WHICH rotary frequencies belong to the (t, h, w) axes; without them an interleaved-MRoPE + * consumer has nothing but a guess, so they are read next to the flag (both spellings) and a model that + * declares mrope without them is refused by the consumer rather than assumed. */ + { + const char* blocks[2] = {value_for_top(json, "rope_scaling"), value_for_top(json, "rope_parameters")}; + for (int b = 0; b < 2 && s->n_mrope_section == 0; b++) { + const char* blk = blocks[b]; + if (!blk || *blk != '{') continue; + const char* at = strstr(blk, "mrope_section"); + if (!at) continue; + at = strchr(at, '['); + if (!at) continue; + int got = 0; + const char* p2 = at + 1; + while (*p2 && *p2 != ']' && got < 3) { + if (*p2 >= '0' && *p2 <= '9') { + char* end = NULL; + s->mrope_section[got++] = (int)strtol(p2, &end, 10); + p2 = end; /* as above: let strtol move the cursor, not the loop */ + } else { + p2++; + } + } + s->n_mrope_section = got; + } + } /* Interleaved MRoPE is declarable in two spellings and only one was read here. FastFlowLM/OGA packs and * Qwen3.5 nest the rotary block under `rope_parameters` (the block below); a Hugging Face `qwen3_vl` config * (Qwen3-VL, engine #1311/TQVL) declares the same thing under `rope_scaling`. Reading only the first meant diff --git a/tests/q4nx_mrope_test.c b/tests/q4nx_mrope_test.c index 7fe900173..e8a42bfbf 100644 --- a/tests/q4nx_mrope_test.c +++ b/tests/q4nx_mrope_test.c @@ -97,6 +97,32 @@ int main(void) { q4nx_semantics_from_config(cfg, dir, &s); check(s.mrope == 0, "mrope_interleaved:false stays plain"); + /* 5. The interleaved axis rule (engine #1498). The sections are stride-3 windows, not contiguous runs: at + * [24, 20, 20] there must be 24 t, 20 h and 20 w frequencies out of 64, and the h/w windows must sit on + * i % 3 == 1 / 2 and stop at 3 * section[dim]. A contiguous split puts 40 of the 64 on the wrong axis. */ + { + int sec[3] = {24, 20, 20}; + int count[3] = {0, 0, 0}; + for (int i = 0; i < 64; i++) count[q4nx_mrope_axis(i, 3, sec)]++; + check(count[0] == 24 && count[1] == 20 && count[2] == 20, "interleaved [24,20,20] -> 24 t / 20 h / 20 w"); + check(q4nx_mrope_axis(1, 3, sec) == 1 && q4nx_mrope_axis(4, 3, sec) == 1, "i%3==1 is the h window"); + check(q4nx_mrope_axis(2, 3, sec) == 2 && q4nx_mrope_axis(5, 3, sec) == 2, "i%3==2 is the w window"); + check(q4nx_mrope_axis(0, 3, sec) == 0 && q4nx_mrope_axis(3, 3, sec) == 0, "i%3==0 stays t"); + check(q4nx_mrope_axis(60, 3, sec) == 0, "past 3 * section[1] the h window closes (i=60 is t)"); + check(q4nx_mrope_axis(61, 3, sec) == 0 && q4nx_mrope_axis(62, 3, sec) == 0, + "past 3 * section[2] the w window closes too"); + /* the contiguous mapping this replaces, for the record: 43 of 64 differ (measured, and the number the + * reviewer gave for engine #1498) */ + int diff = 0; + for (int i = 0; i < 64; i++) { + int old = i < 24 ? 0 : (i < 44 ? 1 : 2); + if (old != q4nx_mrope_axis(i, 3, sec)) diff++; + } + check(diff == 43, "the contiguous reading differs on 43 of 64 frequencies"); + /* and with no sections nothing rotates off the t plane */ + check(q4nx_mrope_axis(1, 0, sec) == 0, "no mrope_section -> plain (t) axis"); + } + if (failures) { printf("q4nx_mrope: %d failure(s)\n", failures); return 1; } printf("q4nx_mrope: ok\n"); return 0; diff --git a/tools/export_qwen3vl_vision.py b/tools/export_qwen3vl_vision.py index 94f41f7a6..7ae2b52f1 100755 --- a/tools/export_qwen3vl_vision.py +++ b/tools/export_qwen3vl_vision.py @@ -130,6 +130,10 @@ def export(hf_dir: str, out_dir: str, opset: int): # a fixed bucket: the patch count is baked in (see dynamo=False below) opset_version=opset, do_constant_folding=True, + # External data: the tower is 1.6 GB of fp32 weights, and a single inline protobuf of that size is where + # ONNX tooling starts failing ("Wire format was corrupt" from the quantiser's pre-processing). Written as + # `.onnx` + `.data`, which is also the shape AMD's own vision packs ship. + external_data=True, # The legacy (TorchScript) exporter, not torch.export: the tower's positional interpolation loops over # the *values* of image_grid_thw (`torch.linspace(0, num_grid_per_side-1, h)` per image), which # torch.export cannot symbolic-shape. This makes the exported graph a FIXED BUCKET: the grid is baked @@ -137,6 +141,20 @@ def export(hf_dir: str, out_dir: str, opset: int): # noted in docs/npu.md. The owner's order allows fixed buckets, so this is the shipped shape. dynamo=False, ) + # Split the weights out by hand when the exporter kept them inline: the legacy exporter ignores + # external_data=, and a 1.6 GB inline protobuf is where the quantiser's pre-processing fails to parse. + try: + import onnx + big = os.path.getsize(path) > 512 * 1024 * 1024 + if big and not os.path.exists(path + ".data"): + m = onnx.load(path) + onnx.save_model(m, path, save_as_external_data=True, all_tensors_to_one_file=True, + location=os.path.basename(path) + ".data", size_threshold=1024) + del m + print(f"split weights to {os.path.basename(path)}.data " + f"({os.path.getsize(path) / 1e6:.1f} MB graph + {os.path.getsize(path + '.data') / 1e6:.1f} MB)") + except ImportError: + pass meta = {"source": os.path.abspath(hf_dir), "opset": opset, "hidden": int(vc.hidden_size), "out_hidden": int(vc.out_hidden_size), "patch": patch, "temporal_patch": int(vc.temporal_patch_size), "merge": merge, "depth": depth, "heads": int(vc.num_heads), @@ -210,15 +228,57 @@ def hf_dir_of(fp32_dir: str) -> str: return meta.get("source", "") -def quantize(fp32_dir: str, out_dir: str, calib: int): +def bench(out_dir: str, images: int, providers: str): + """Seconds per image through the graph, on the providers named (e.g. "CPUExecutionProvider"). + + This is gate 3's other half: the CPU projector in llama.cpp measures 16.97 s/image on this box, and a row here is + what our graph costs on the same machine. Not an accuracy row -- `check` is. + """ + import onnxruntime as ort + import time + + meta = json.load(open(os.path.join(out_dir, "vision_fp32.json"))) + bucket, patch, temporal = int(meta["bucket_patches"]), int(meta["patch"]), int(meta["temporal_patch"]) + provs = [p.strip() for p in providers.split(",") if p.strip()] + sess = ort.InferenceSession(os.path.join(out_dir, "vision_fp32.onnx"), providers=provs) + rng = np.random.default_rng(11) + pv = make_image(rng, bucket, bucket, patch, temporal) + grid = np.array([[1, bucket, bucket]], dtype=np.int64) + t0 = time.time() + sess.run(None, {"pixel_values": pv, "image_grid_thw": grid}) # warm-up (graph + arena) + warm = time.time() - t0 + times = [] + for _ in range(images): + pv = make_image(rng, bucket, bucket, patch, temporal) + t0 = time.time() + sess.run(None, {"pixel_values": pv, "image_grid_thw": grid}) + times.append(time.time() - t0) + best, med = min(times), sorted(times)[len(times) // 2] + print(f"providers={sess.get_providers()} images={images} warm-up {warm:.2f}s | " + f"per image best {best:.2f}s median {med:.2f}s | {1.0 / best:.3f} images/s") + print(f"cpu bar to beat (llama.cpp CPU projector, measured): 16.97 s/image") + return 0 + + +def quantize(fp32_dir: str, out_dir: str, calib: int, stream: bool = False, clip_percentile: float = 99.9): """QDQ int8, A8W8 per-channel -- the shape our ORT EP runs (Quark-style A8W8). Activations are quantised per tensor (QDQ), weights per output channel, both int8, and `image_grid_thw` is left alone (it is an integer input, not an activation). The calibration set is deterministic images at the exported bucket, so the row is reproducible; a real deployment would calibrate on its own data. """ + if stream: + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # tools/vision_qdq.py sits beside this file + import vision_qdq + + # same contract as the ORT path below: 1 when the quantised graph misses the cosine bar, so a failing graph can + # never exit 0 + return vision_qdq.quantise(os.path.join(fp32_dir, "vision_fp32.onnx"), os.path.join(out_dir, "vision_qdq.onnx"), + calib=calib, clip_percentile=clip_percentile) + import onnxruntime as ort - from onnxruntime.quantization import CalibrationDataReader, QuantFormat, QuantType, quantize_static + from onnxruntime.quantization import CalibrationDataReader, CalibrationMethod, QuantFormat, QuantType, quantize_static + from onnxruntime.quantization.shape_inference import quant_pre_process # kept for the record above meta = json.load(open(os.path.join(fp32_dir, "vision_fp32.json"))) bucket, patch = int(meta["bucket_patches"]), int(meta["patch"]) @@ -247,13 +307,23 @@ def rewind(self): os.makedirs(out_dir, exist_ok=True) dst = os.path.join(out_dir, "vision_qdq.onnx") + # Pre-processing first: symbolic shape inference + the standard fusions. Without it ORT quantises a graph whose + # tensors have no inferred ranges at several nodes and the calibration never reaches them — measured as a + # systematic, not a precision, error (cosine 0.22 with it missing). + # No quant_pre_process() here on purpose: with the weights in an external `.data` file it copies the model to a + # temp dir and leaves the data behind ("Data of TensorProto … should be stored in /tmp/pre.quant.*/ + # vision_fp32.onnx.data, but it is not regular file"), and with the weights inline it fails earlier on + # symbolic shape inference. The graph is a fixed bucket with concrete shapes, so the quantiser's own + # pre-processing is not needed; this is recorded rather than worked around with a fragile temp-file dance. + _ = quant_pre_process quantize_static(src, dst, Reader(), quant_format=QuantFormat.QDQ, per_channel=True, + calibrate_method=CalibrationMethod.Percentile, activation_type=QuantType.QInt8, weight_type=QuantType.QInt8, # Only the big projections. Quantising Conv (the patch embed), Add (residuals) and the # norm/softmax paths as well was measured at cosine 0.22 against fp32 -- the graph is a ViT # with per-token dynamic ranges and no QDQ pre-processing, so the sensitive ops stay float. op_types_to_quantize=["MatMul"], - extra_options={"ActivationSymmetric": True, "WeightSymmetric": True}) + extra_options={"ActivationSymmetric": True, "WeightSymmetric": True, "CalibPercentile": 99.9}) out_meta = dict(meta) out_meta.update({"quant": "qdq-a8w8-perchannel-int8", "calib_images": calib}) json.dump(out_meta, open(os.path.join(out_dir, "vision_qdq.json"), "w"), indent=1) @@ -279,7 +349,17 @@ def main(): q = sub.add_parser("quantize") q.add_argument("fp32_dir") q.add_argument("out_dir") - q.add_argument("--calib", type=int, default=16) + q.add_argument("--calib", type=int, default=32) + q.add_argument("--stream", action="store_true", + help="streaming per-tensor QDQ instead of ORT's quantize_static (which needs ~64 GiB peak for this " + "graph: measured by a systemd scope at 64.1 G before the OOM killer; the streaming path peaks " + "at 6.4 G)") + q.add_argument("--clip-percentile", type=float, default=99.9, + help="with --stream: per-tensor |x| percentile used as the activation range") + b = sub.add_parser("bench") + b.add_argument("out_dir") + b.add_argument("--images", type=int, default=3) + b.add_argument("--providers", default="CPUExecutionProvider") c = sub.add_parser("check") c.add_argument("hf_dir") c.add_argument("out_dir") @@ -290,7 +370,9 @@ def main(): export(a.hf_dir, a.out_dir, a.opset) return 0 if a.cmd == "quantize": - return quantize(a.fp32_dir, a.out_dir, a.calib) + return quantize(a.fp32_dir, a.out_dir, a.calib, a.stream, a.clip_percentile) + if a.cmd == "bench": + return bench(a.out_dir, a.images, a.providers) return check(a.hf_dir, a.out_dir, a.images, a.onnx) diff --git a/tools/vision_qdq.py b/tools/vision_qdq.py new file mode 100644 index 000000000..15194cbf9 --- /dev/null +++ b/tools/vision_qdq.py @@ -0,0 +1,257 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Streaming QDQ A8W8 for the vision graph: one weight tensor in memory at a time. + +Used by `export_qwen3vl_vision.py quantize --stream`. + +ORT's quantize_static needs ~64 GiB peak for this 1.6 GB graph (measured: the systemd scope recorded 64.1 G peak before +the OOM killer), which a shared box cannot spare. This does the same job by editing the graph directly: + + * weights: per-output-channel symmetric int8, emitted as QuantizeLinear -> DequantizeLinear pairs; + * activations: per-tensor symmetric int8 in front of every `Gemm` (the projections; attention's activation-times- + activation `MatMul`s are left float, see `matmul_weights`), with the scale calibrated from a few images -- the fp32 + graph is run once with every `Gemm` input exposed as an output, so no intermediate probing library is needed; + * everything else (Conv, norms, softmax, split) stays float. + + vision_qdq.py [--calib N] (normally reached as `export_qwen3vl_vision.py + quantize --stream`) +""" +import argparse +import json +import os +import sys + +import numpy as np +import onnx +from onnx import TensorProto, helper, numpy_helper + + +def matmul_weights(model): + """(node, initializer, per_output_axis) for every layer that multiplies by a WEIGHT initializer. + + The tower's projections are `Gemm(A, W, bias)` with `transB=1` (torch exports Linear that way), so W is [N, K] and + its per-output-channel scale runs along axis 0. The `MatMul` nodes that remain are attention's A@K^T and A@V, + where *both* operands are activations -- quantising those needs both inputs quantised and is deliberately left + float here, since the projections carry the FLOPs. + """ + inits = {i.name: i for i in model.graph.initializer} + out = [] + for n in model.graph.node: + if n.op_type not in ("Gemm", "MatMul") or len(n.input) < 2 or n.input[1] not in inits: + continue + w = numpy_helper.to_array(inits[n.input[1]]) + if w.ndim != 2: + continue + trans_b = 0 + if n.op_type == "Gemm": + for at in n.attribute: + if at.name == "transB": + trans_b = int(at.i) + # reduce over K: axis 1 for W [N, K] (Gemm transB=1, the torch Linear export), axis 0 for W [K, N] + out.append((n, inits[n.input[1]], 1 if (n.op_type == "Gemm" and trans_b == 1) else 0)) + return out, inits + + +def expose_outputs(model, names): + have = {o.name for o in model.graph.output} + for nm in names: + if nm in have: + continue + model.graph.output.append(helper.make_tensor_value_info(nm, TensorProto.FLOAT, None)) + + +def make_image(rng, gh, gw, patch, temporal): + """A deterministic image at the exported bucket: two gradients plus a chequerboard, in [0, 1] before normalising.""" + H, W = gh * patch, gw * patch + yy, xx = np.mgrid[0:H, 0:W] + img = np.zeros((3, H, W), dtype=np.float32) + img[0] = (yy / max(H - 1, 1)) * 0.8 + 0.1 + img[1] = (xx / max(W - 1, 1)) * 0.8 + 0.1 + img[2] = ((yy // 32 + xx // 32) % 2) * 0.7 + 0.1 + img += rng.normal(0, 0.02, img.shape).astype(np.float32) + img = np.clip(img, 0, 1) + blocks = img.reshape(3, gh, patch, gw, patch).transpose(1, 3, 0, 2, 4) + if temporal != 1: + blocks = np.repeat(blocks[:, :, None], temporal, axis=2) + return blocks.reshape(gh * gw, 3 * temporal * patch * patch).astype(np.float32) + + +def run_calibration(path, names, images, bucket, patch, temporal, calib_pv="", clip_pct=99.9): + import onnxruntime as ort + + sess = ort.InferenceSession(path, providers=["CPUExecutionProvider"]) + got = {nm: 0.0 for nm in names} # the max over images of each image's p99.9 |x| (see --clip-percentile) + feeds = [] + if calib_pv: # the real picture first: synthetic gradients do not bound a photo + feeds.append(np.load(calib_pv).astype(np.float32)) + rng = np.random.default_rng(3) + for _ in range(images): + feeds.append(make_image(rng, bucket, bucket, patch, temporal)) + for pv in feeds: + grid = np.array([[1, bucket, bucket]], dtype=np.int64) + outs = sess.run(names, {"pixel_values": pv, "image_grid_thw": grid}) + for nm, arr in zip(names, outs): + # a single outlier must not set the whole tensor's scale: one activation here has absmax 1083 while the + # mass of that tensor sits near zero, and max-based scales zero it out (measured: 103/104 tensors diverge, + # cosine -0.03). A high percentile keeps the dynamic range the mass actually uses. + a_ = np.abs(arr.reshape(-1)).astype(np.float32) + got[nm] = max(got[nm], float(np.percentile(a_, clip_pct))) + return got + + +def quantise(src, dst, calib=6, clip_percentile=99.9, calib_pv="", bucket=28, patch=16, temporal=2, + bar=0.999): + """Quantise `src` into `dst` (+ `dst`.data). Returns the number of quantised weights.""" + a = argparse.Namespace(src=src, dst=dst, calib=calib, bucket=bucket, patch=patch, temporal=temporal, + no_act_qdq=False, calib_pv=calib_pv, clip_percentile=clip_percentile, bar=bar) + return _run(a) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("src") + ap.add_argument("dst") + ap.add_argument("--calib", type=int, default=6) + ap.add_argument("--no-act-qdq", action="store_true", help="weights only (W8A32): isolates which side loses the answer") + ap.add_argument("--bucket", type=int, default=28) + ap.add_argument("--patch", type=int, default=16) + ap.add_argument("--temporal", type=int, default=2) + ap.add_argument("--calib-pv", default="", help="calibrate on these pixels (a .npy), so no PIL is needed here") + ap.add_argument("--bar", type=float, default=0.999, help="self-check cosine bar against the fp32 graph") + ap.add_argument("--clip-percentile", type=float, default=99.9, + help="per-tensor |x| percentile used as the activation range (100 = plain max, which loses a tensor " + "to one outlier: measured 103/104 tensors diverge)") + return _run(ap.parse_args()) + + +def _run(a): + os.makedirs(os.path.dirname(os.path.abspath(a.dst)), exist_ok=True) + model = onnx.load(a.src) # ~2 GB in memory; the weights are not copied per tensor + pairs, inits = matmul_weights(model) + print(f"{len(pairs)} MatMul weights to quantise") + + # --- calibrate the activations: expose each MatMul's A input, run a few images, take the absmax + names = sorted({n.input[0] for n, _, _ in pairs}) + calib_model = onnx.load(a.src) + expose_outputs(calib_model, names) + cpath = a.dst + ".calib.onnx" + onnx.save(calib_model, cpath) + del calib_model + act_absmax = run_calibration(cpath, names, a.calib, a.bucket, a.patch, a.temporal, a.calib_pv, a.clip_percentile) + os.unlink(cpath) + vals = sorted(act_absmax.values()) + tiny = [nm for nm, v in act_absmax.items() if v < 1e-3] + print(f"calibrated {len(act_absmax)} activation tensors on {a.calib} images: " + f"absmax min {vals[0]:.4g} median {vals[len(vals)//2]:.4g} max {vals[-1]:.4g}; " + f"{len(tiny)} below 1e-3" + (f" (e.g. {', '.join(tiny[:4])})" if tiny else "")) + + # --- rewrite: per-channel int8 weights, per-tensor int8 activations, QDQ pairs + new_inits, replaced = [], set() + act_scale = {} if a.no_act_qdq else {nm: max(v / 127.0, 1e-8) for nm, v in act_absmax.items()} + # QuantizeLinear's output type comes from its zero point, and its default is UINT8 -- which silently clamps every + # negative activation to 0. An int8 zero point makes the codes int8 and is what the DequantizeLinear below expects + # (measured: without it the first Gemm's activations lose all negatives, 103/104 tensors diverge, cosine -0.03). + for nm, sc in act_scale.items(): + new_inits.append(numpy_helper.from_array(np.array([sc], dtype=np.float32), name=nm + "_qdq_scale")) + new_inits.append(numpy_helper.from_array(np.zeros([1], dtype=np.int8), name=nm + "_qdq_zero")) + + # quantise each weight into a new initializer pair, remembering what each Gemm's inputs become + gemm_rewrite = {} + for n, init, axis in pairs: + w = numpy_helper.to_array(init) # one tensor at a time; the 64 GiB ORT needs for this is why + if w.ndim != 2 or 0 in w.shape: + continue + red = (1,) if axis == 1 else (0,) + cols = w.shape[0] if axis == 1 else w.shape[1] + amax = np.abs(w).max(axis=red, keepdims=True) + scale = np.maximum(amax / 127.0, 1e-8).astype(np.float32) + q = np.clip(np.rint(w / scale), -127, 127).astype(np.int8) + base = init.name + "_qdq" + new_inits.append(numpy_helper.from_array(q, name=base + "_q")) + new_inits.append(numpy_helper.from_array(scale.reshape(cols), name=base + "_s")) + # the codes keep the weight's layout, so the per-channel axis is the OTHER one (scale runs along N) + gemm_rewrite[id(n)] = (n, base, 1 - red[0]) + replaced.add(init.name) + del w, q, amax, scale + + # rebuild the node list in topological order, putting each QDQ pair immediately before its consumer + act_emitted = set() + out_nodes = [] + for n in model.graph.node: + rw = gemm_rewrite.get(id(n)) + if rw: + _, base, dq_axis = rw + a_name = n.input[0] + if a_name in act_scale and a_name not in act_emitted: + act_emitted.add(a_name) + out_nodes.append(helper.make_node("QuantizeLinear", + [a_name, a_name + "_qdq_scale", a_name + "_qdq_zero"], + [a_name + "_qdq_q"], name=a_name + "_qdq_ql")) + out_nodes.append(helper.make_node("DequantizeLinear", [a_name + "_qdq_q", a_name + "_qdq_scale"], + [a_name + "_qdq_dq"], name=a_name + "_qdq_dql")) + for n2, _, _ in pairs: + if n2.input[0] == a_name: + n2.input[0] = a_name + "_qdq_dq" + out_nodes.append(helper.make_node("DequantizeLinear", [base + "_q", base + "_s"], [base + "_dq"], + name=base, axis=dq_axis)) + for i, inp in enumerate(n.input): + if inp in replaced: + n.input[i] = base + "_dq" + out_nodes.append(n) + + keep = [i for i in model.graph.initializer if i.name not in replaced] + model.graph.ClearField("initializer") + model.graph.initializer.extend(keep + new_inits) + model.graph.ClearField("node") + model.graph.node.extend(out_nodes) + onnx.checker.check_model(model) + # a previous run's weight blob is appended to, not replaced, by save_model with external data + for stale in (a.dst, a.dst + ".data"): + if os.path.exists(stale): + os.unlink(stale) + onnx.save_model(model, a.dst, save_as_external_data=True, all_tensors_to_one_file=True, + location=os.path.basename(a.dst) + ".data", size_threshold=1024) + print(f"wrote {a.dst} ({os.path.getsize(a.dst) / 1e6:.1f} MB) + .data " + f"({os.path.getsize(a.dst + '.data') / 1e6:.1f} MB); {len(replaced)} weights, {len(act_emitted)} activations") + return self_check(a.src, a.dst, images=3, bar=a.bar) + + +def self_check(fp32_path, qdq_path, images=3, bar=0.999): + """Cosine of every output against the fp32 graph on the same images; returns 0 if it clears `bar`, 1 if not. + + The ORT path in export_qwen3vl_vision.py gates itself the same way, and the point is that a graph below the bar must + not exit 0: the A8W8 output measures 0.61-0.91, so this path has to be able to fail. + """ + import onnxruntime as ort + + a = ort.InferenceSession(fp32_path, providers=["CPUExecutionProvider"]) + b = ort.InferenceSession(qdq_path, providers=["CPUExecutionProvider"]) + rng = np.random.default_rng(11) + worst = 1.0 + for _ in range(images): + pv = make_image(rng, 28, 28, 16, 2) + feed = {"pixel_values": pv, "image_grid_thw": np.array([[1, 28, 28]], dtype=np.int64)} + for x, y in zip(a.run(None, feed), b.run(None, feed)): + x, y = x.reshape(-1).astype(np.float64), y.reshape(-1).astype(np.float64) + worst = min(worst, float(np.dot(x, y) / (np.linalg.norm(x) * np.linalg.norm(y) + 1e-12))) + ok = worst >= bar + print(f"self-check: worst cosine {worst:.6f} against the fp32 graph on {images} images " + f"({'PASS' if ok else 'FAIL'}; the bar is {bar})") + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main())