Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1267,6 +1267,18 @@ if(ONEBIT_NPU)
if(ONEBIT_NPU)
# The decoder's precomputed-embedding entry point (npu/forward/embeds_equiv_probe.cpp): rows built from the
# model's own table must give byte-identical logits to the token-id path, on the host and on the array.
# The three-plane (MRoPE) position input: equal planes must be byte-identical to plain RoPE, different planes
# must move the logits (npu/forward/mrope_planes_probe.cpp).
# The wiring a vision-conditioned prompt needs on the engine side: vision rows + 3-plane positions + deepstack
# sets through step_batch_embeds(), then a greedy decode (npu/forward/hybrid_prompt_probe.cpp).
add_executable(hybrid_prompt_probe npu/forward/hybrid_prompt_probe.cpp)
target_include_directories(hybrid_prompt_probe PRIVATE ${CMAKE_SOURCE_DIR}/npu
${CMAKE_SOURCE_DIR}/npu/forward/include ${CMAKE_SOURCE_DIR}/npu/forward/engine-src)
target_link_libraries(hybrid_prompt_probe PRIVATE onebit_npu_forward onebit_npu_rt)
add_executable(mrope_planes_probe npu/forward/mrope_planes_probe.cpp)
target_include_directories(mrope_planes_probe PRIVATE ${CMAKE_SOURCE_DIR}/npu
${CMAKE_SOURCE_DIR}/npu/forward/include ${CMAKE_SOURCE_DIR}/npu/forward/engine-src)
target_link_libraries(mrope_planes_probe PRIVATE onebit_npu_forward onebit_npu_rt)
add_executable(embeds_equiv_probe npu/forward/embeds_equiv_probe.cpp)
target_include_directories(embeds_equiv_probe PRIVATE ${CMAKE_SOURCE_DIR}/npu
${CMAKE_SOURCE_DIR}/npu/forward/include ${CMAKE_SOURCE_DIR}/npu/forward/engine-src)
Expand Down
175 changes: 175 additions & 0 deletions npu/forward/hybrid_prompt_probe.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,175 @@
// Copyright 2026 bong-water-water-bong
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// A vision-conditioned prompt through the decoder (engine#1311/TQVL): the wiring gate 2 needs, on the engine side.
//
// Reads what the vision half produced -- the prompt's token ids, the image rows' embeddings, the 3-plane positions and
// the deepstack sets -- and runs the prefill through step_batch_embeds(), then greedily decodes N tokens. Text rows'
// embeddings come from the model's own table (embed_row), so the only foreign rows are the image ones.
//
// hybrid_prompt_probe <model.q4nx> <kernels dir> <input dir> [--decode N]
//
// <input dir> holds ids.i32, positions.i32 (3 x rows), pooled.f32 (rows_img x H), deepstack.f32 (sets x rows_img x H)
// and meta.json {"image_lo": i, "image_hi": j}. Build them with the vision graph (tools/export_qwen3vl_vision.py)
// plus the tokenizer; this probe does not touch a tokenizer.
#include "q4nx_forward.h"

#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <memory>
#include <string>
#include <algorithm>
#include <utility>
#include <vector>

template <class T>
static bool read_vec(const std::string& path, std::vector<T>& out) {
std::ifstream f(path, std::ios::binary);
if (!f) return false;
f.seekg(0, std::ios::end);
const auto n = (size_t)f.tellg() / sizeof(T);
f.seekg(0);
out.resize(n);
f.read((char*)out.data(), (std::streamsize)(n * sizeof(T)));
return (bool)f;
}

int main(int argc, char** argv) {
if (argc < 4) {
std::fprintf(stderr, "usage: %s <model.q4nx> <kernels dir> <input dir> [--decode N]\n", argv[0]);
return 2;
}
int decode = 8, top_k_n = 0;
for (int i = 4; i + 1 < argc; i++)
if (!std::strcmp(argv[i], "--decode")) decode = std::atoi(argv[i + 1]);
else if (!std::strcmp(argv[i], "--top-k") && i + 1 < argc) top_k_n = std::atoi(argv[++i]);
const std::string dir = argv[3];

Q4nxNpuForward f;
std::unique_ptr<xrt::device> dev;
bool ok;
if (getenv("NPU_INFER_HOST_GEMM")) {
ok = f.init_host(argv[1], argv[2]);
} else {
dev = std::make_unique<xrt::device>(0);
ok = f.init(*dev, argv[1], argv[2]);
}
if (!ok) { std::fprintf(stderr, "init: %s\n", f.last_error()); return 1; }
const int H = 0; // read from a row, below
(void)H;

std::vector<int32_t> ids, positions;
std::vector<float> pooled, deepstack;
if (!read_vec(dir + "/ids.i32", ids) || !read_vec(dir + "/positions.i32", positions) ||
!read_vec(dir + "/pooled.f32", pooled) || !read_vec(dir + "/deepstack.f32", deepstack)) {
std::fprintf(stderr, "cannot read the input files from %s\n", dir.c_str());
return 2;
}
// meta.json is tiny; the two numbers are read by position so the probe needs no JSON library
int meta[2] = {-1, -1};
{
std::ifstream m(dir + "/meta.txt");
if (m) m >> meta[0] >> meta[1]; // image_lo image_hi (one line each is fine)
}
// image_lo == image_hi is the text-only control: no image rows, nothing to inject.
if (meta[0] < 0 || meta[1] < meta[0]) { std::fprintf(stderr, "bad meta (image_lo image_hi)\n"); return 2; }
const int rows = (int)ids.size();
std::vector<float> one;
f.embed_row(11, one);
const size_t Hd = one.size();
const size_t rows_img = size_t(meta[1] - meta[0]);
if (pooled.size() != rows_img * Hd || positions.size() != size_t(3) * size_t(rows)) {
std::fprintf(stderr, "shapes: pooled %zu (want %zu), positions %zu (want %zu)\n", pooled.size() / Hd,
rows_img, positions.size(), size_t(3) * rows);
return 2;
}
const int n_ds = f.n_deepstack_layers();
if (n_ds > 0 && deepstack.size() != size_t(n_ds) * rows_img * Hd) {
std::fprintf(stderr, "deepstack %zu, want %d sets x %zu rows x %zu\n", deepstack.size(), n_ds, rows_img, Hd);
return 2;
}

// embedding rows: the model's own for text positions, the vision graph's for the image span
std::vector<float> embeds;
embeds.reserve(size_t(rows) * Hd);
for (int r = 0; r < rows; r++) {
if (r >= meta[0] && r < meta[1]) {
const float* src = pooled.data() + size_t(r - meta[0]) * Hd;
embeds.insert(embeds.end(), src, src + Hd);
} else {
f.embed_row(ids[size_t(r)], one);
embeds.insert(embeds.end(), one.begin(), one.end());
}
}
Q4nxNpuForward::HybridInputs in;
in.position_ids = &positions;
// A text-only prompt (the control) has no image rows, so there is no span to add deepstack sets at.
if (n_ds > 0 && rows_img > 0) {
in.deepstack = deepstack.data();
in.deepstack_sets = n_ds;
in.image_lo = meta[0];
in.image_hi = meta[1];
}

std::vector<float> logits;
f.reset();
if (!f.step_batch_embeds(embeds, 0, logits, in)) { std::fprintf(stderr, "prefill: %s\n", f.last_error()); return 1; }
auto argmax = [](const std::vector<float>& v) {
int b = 0;
for (size_t i = 1; i < v.size(); i++)
if (v[i] > v[(size_t)b]) b = (int)i;
return b;
};
// --top-k N prints the top-N logits wherever an argmax is taken, so a decode step can be diffed against the
// reference number for number instead of token for token (two decoders that agree on the argmax can still differ by
// a lot underneath, and that is what decides whether the next step is off).
auto top_k = [&](const char* where) {
if (top_k_n <= 0) return;
std::vector<std::pair<float, int>> v;
v.reserve(logits.size());
for (size_t i = 0; i < logits.size(); i++) v.emplace_back(logits[i], (int)i);
const size_t k = std::min((size_t)top_k_n, v.size());
std::partial_sort(v.begin(), v.begin() + (long)k, v.end(),
[](const std::pair<float, int>& a, const std::pair<float, int>& b) { return a.first > b.first; });
std::printf("top%d %s:", top_k_n, where);
for (size_t i = 0; i < k; i++) std::printf(" %d:%.4f", v[i].second, v[i].first);
std::printf("\n");
std::fflush(stdout);
};
std::vector<int> out;
top_k("prefill");
out.push_back(argmax(logits));
int pos = int(positions[0 * rows + rows - 1]) + 1; // the generation step's ROTARY position (delta applied)
// ...and the slot the token lands in, which is the prompt's length, not that position. They coincide for a text
// prompt and do not after an image: the image rows walk a small h/w grid, so `pos` is far below the cache length and
// a step that used it as the window read only the first `pos + 1` rows -- losing the question and never attending to
// what it had just generated (the review of engine #1498 caught this; it is why every image answer was bad).
for (int i = 1; i < decode; i++) {
const int cache_pos = rows + i - 1;
if (!f.step(out.back(), pos + i - 1, logits, cache_pos)) {
std::fprintf(stderr, "step: %s\n", f.last_error());
return 1;
}
top_k(i == 1 ? "step1" : "step");
out.push_back(argmax(logits));
}
std::printf("prompt %d tokens (%zu image rows), deepstack %d set(s)\n", rows, rows_img, in.deepstack_sets);
std::printf("generated %d token ids:", (int)out.size());
for (int t : out) std::printf(" %d", t);
std::printf("\n");
return 0;
}
42 changes: 40 additions & 2 deletions npu/forward/include/q4nx_forward.h
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,12 @@ class Q4nxNpuForward {
bool extend_context(int n_tokens);

// Run one token at absolute position pos; fills logits (size NV).
bool step(int token, int pos, std::vector<float>& logits);
// `pos` is the ROTARY position; `cache_pos` is the slot the new K/V lands in, which is the only thing that
// decides how many cached rows attention reads. They are the same number only while positions equal indices:
// after an image they are not (an image's rows walk a small h/w grid, so the MRoPE position is far below the
// cache length), and using the rotary position as the window dropped the prompt's tail -- the question -- and
// every token after the first never saw what it had generated. -1 keeps the old behaviour for text prompts.
bool step(int token, int pos, std::vector<float>& logits, int cache_pos = -1);

// Batched prefill: run the whole prompt in one pass so each projection's
// weight stream is read once per batch instead of once per token. The kernels
Expand All @@ -148,6 +153,29 @@ class Q4nxNpuForward {
// The per-token fallback path (MLA/MoE/GDN/... models) needs ids and is refused by name.
bool step_batch_embeds(const std::vector<float>& embeds, int start_pos,
std::vector<float>& logits);
// What a vision-conditioned prompt adds on top of the embeddings (engine#1311/TQVL; plan P4's `rope.planes`
// and a per-layer feature injection):
// position_ids 3 x rows, plane-major (t, h, w) — the layout the reference's `get_rope_index` produces and
// AMD's `text.onnx` takes. Text rows have all three equal, which is bit-identical to plain RoPE
// (measured: 0 difference over positions 0..262143; image rows differ per axis and that is the
// whole point). A pack that declares mrope without sections is refused by name.
// deepstack sets x rows x H, one set per `vision_config.deepstack_visual_indexes` entry (Qwen3-VL: 5/11/17);
// set s is added to the hidden state when the layer loop reaches deepstack_layers[s]. This is how
// the reference injects the vision tower's extra features, and an answer cannot match it without.
struct HybridInputs {
const std::vector<int>* position_ids = nullptr;
const float* deepstack = nullptr;
int deepstack_sets = 0;
// The rows the deepstack sets belong to: the image span. They are added THERE, not everywhere -- a set has
// rows_img rows, so adding it to every row of the prompt would both be the wrong function and read past the
// buffer (the first version of this did exactly that and produced garbage tokens).
int image_lo = -1;
int image_hi = -1;
};
const int* deepstack_layers() const { return sem_.deepstack_layers; }
int n_deepstack_layers() const { return sem_.n_deepstack; }
bool step_batch_embeds(const std::vector<float>& embeds, int start_pos, std::vector<float>& logits,
const HybridInputs& in);

// The same batched pass, but returning the logits of EVERY row (M x NV, row-major) instead of only
// the last one. Speculative decoding needs them: a draft token is accepted only if the model's own
Expand Down Expand Up @@ -231,7 +259,7 @@ class Q4nxNpuForward {
// The default for the above when NPU_FFN_CACHE says nothing: on for a GDN
// model on the NPU route, with a size guard. See the definition.
bool ffn_cache_default() const;
bool attention(const std::vector<float>& qkv, int pos, std::vector<float>& ctx,
bool attention(const std::vector<float>& qkv, int pos, const int* pos3, std::vector<float>& ctx,
int head_dim, int n_kv_heads, float theta, int rotary_dim,
bool proportional, int window, int kv_layer, bool append_kv,
int kv_len = 0);
Expand Down Expand Up @@ -354,6 +382,16 @@ class Q4nxNpuForward {
std::vector<uint16_t> embed_; // [NV*H] bf16
// Set only while step_batch_embeds() runs: the row source for the batched body's embedding fill.
const float* inject_embeds_ = nullptr;
// Set only while step_batch_embeds(..., position_ids, ...) runs: 3 x rows, plane-major, or null.
const int* pos3_ = nullptr;
int pos3_rows_ = 0;
// Set only while a step_batch_embeds() call carries deepstack sets: sets x rows x H.
const float* deepstack_ = nullptr;
int deepstack_rows_ = 0;
int deepstack_lo_ = -1, deepstack_hi_ = -1;
// Which plane a rotary frequency belongs to under interleaved MRoPE. The rule lives in q4nx_semantics.h as
// q4nx_mrope_axis() so the C unit test can pin it without building a forward pass.
int mrope_plane(int freq) const { return q4nx_mrope_axis(freq, sem_.n_mrope_section, sem_.mrope_section); }
// Some models store the embedding QUANTIZED (Gemma4-E2B/E4B: dtype I8), and
// reading it as bf16 yields ~zero - which invalidated the whole forward and
// the tied LM head (all-zero logits). When the dtype is not BF16 the
Expand Down
34 changes: 34 additions & 0 deletions npu/forward/include/q4nx_semantics.h
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,17 @@ typedef struct {
/* Interleaved/multimodal RoPE variant that this forward does not implement.
* Source: rope_parameters.mrope_interleaved (or rope_type != "default"). */
int mrope;
/* MVROPE sections: how many of the rotary frequencies belong to each of the (t, h, w) axes. Read from
* rope_scaling.mrope_section (Hugging Face) or rope_parameters.mrope_section (OGA/Qwen3.5); zeros and
* n_mrope_section == 0 mean "declared mrope without sections", which a consumer must refuse rather than
* guess a layout for. */
int mrope_section[3];
int n_mrope_section;
/* DEEPSTACK: which decoder layers take one of the vision tower's extra feature sets. Read from
* vision_config.deepstack_visual_indexes (Qwen3-VL: [5, 11, 17]); the consumer adds set s to the hidden state at
* layer deepstack_layers[s]. Empty means the pack has none. */
int deepstack_layers[8];
int n_deepstack;
/* RoPE is configured PER LAYER TYPE (Gemma4 nests rope_parameters.
* full_attention and .sliding_attention with different thetas and types).
* Source: presence of rope_parameters.{full,sliding}_attention sub-blocks. */
Expand Down Expand Up @@ -193,6 +204,29 @@ void q4nx_semantics_default(Q4nxSemantics* s);
int q4nx_semantics_from_config(const char* json_path, const char* model_dir,
Q4nxSemantics* s);

/* Which axis a rotary frequency belongs to under INTERLEAVED MRoPE.
*
* The three `mrope_section` values are not contiguous runs of frequencies. The reference builds cos/sin for all three
* planes and recomposes them: start from the t plane and, for each of h and w, overwrite every third frequency with that
* plane's value over a stride-3 window of length `section[dim] * 3` (transformers
* `Qwen3VLTextRotaryEmbedding.recomposition_frequencies`: `idx = slice(offset, mrope_section[dim] * 3, 3)`).
*
* axis(i) = 1 if i % 3 == 1 && i < 3 * section[1]
* = 2 if i % 3 == 2 && i < 3 * section[2]
* = 0 otherwise
*
* For [24, 20, 20] that is 24 t / 20 h / 20 w frequencies, and 43 of the 64 sit where a contiguous split would not put
* them -- which is why treating the sections as contiguous runs left every image prompt's answer bad while all
* equal-plane (text-only) prompts stayed exact. Frequency index is the index within head_dim/2.
*/
static inline int q4nx_mrope_axis(int freq, int n_section, const int* section) {
if (n_section < 3 || !section) return 0;
const int m = freq % 3;
if (m == 1 && freq < 3 * section[1]) return 1;
if (m == 2 && freq < 3 * section[2]) return 2;
return 0;
}

/* Human-readable one-line dump of every flag with its source field. */
void q4nx_semantics_dump(const Q4nxSemantics* s);

Expand Down
Loading
Loading