diff --git a/CMakeLists.txt b/CMakeLists.txt index dfcf68f2a..e867c07ba 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2256,6 +2256,15 @@ if(ONEBIT_NPU_HOST AND NOT ONEBIT_NPU AND NOT WIN32 AND Python3_Interpreter_FOUN COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/batch_check_state_ci.py $ qwen3_5) set_tests_properties(batch_check_state_qwen35 PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 120) + # engine#1786: the bidirectional gemma3 encoder (Embedding-Gemma-300M-NPU2's layout: BF16 projections, unprefixed + # names, pre-scaled embedding, raw norms, a sliding and a full layer) through embed_pooled(), against a numpy + # encoder on the same weights, and an over-long input refused by name. + add_executable(embed_pooled_test tests/embed_pooled_test.cpp) + target_link_libraries(embed_pooled_test PRIVATE onebit_npu_forward_ci) + add_dependencies(onebit embed_pooled_test) + add_test(NAME embed_forward + COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/embed_forward_ci.py $) + set_tests_properties(embed_forward PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 120) endif() # Private NPU routes (docs/npu.md, "Private routes"): closed-source add-ons in a checkout of diff --git a/app/embed_serve.cpp b/app/embed_serve.cpp index 3ac605026..ecfa5f938 100644 --- a/app/embed_serve.cpp +++ b/app/embed_serve.cpp @@ -29,6 +29,13 @@ #ifdef ONEBIT_NPU #include "embed_npu.h" +#include "../npu/forward/include/q4nx_forward.h" +#include "npu_device_lock.h" +#include "tokenizer.h" +#include "rt.h" +#include +#include +#include #endif namespace fs = std::filesystem; @@ -41,11 +48,36 @@ bool container(const std::string& path) { return fs::is_directory(path, ec) && fs::is_regular_file(fs::path(path) / "model.embed", ec); } +bool forward_encoder(const std::string& path) { +#ifdef ONEBIT_NPU + std::error_code ec; + const fs::path d(path); + if (!fs::is_directory(d, ec) || !fs::is_regular_file(d / "model.q4nx", ec)) return false; + std::ifstream f(d / "config.json"); + if (!f) return false; + try { + const json c = json::parse(f); + return c.value("use_bidirectional_attention", false); + } catch (const std::exception&) { + return false; + } +#else + (void)path; + return false; +#endif +} + struct Backend::Impl { embed::Model model; #ifdef ONEBIT_NPU embed::NpuGemm npu; bool npu_installed = false; + // engine#1786: the q4nx encoder kind. One forward, so requests are serialised. + std::unique_ptr dev; + std::unique_ptr fwd; + std::unique_ptr tok; + int bos = -1, eos = -1, cap = 0, lock_fd = -1; + std::mutex mu; #endif }; @@ -55,6 +87,66 @@ bool Backend::load(const std::string& dir, const std::string& device, std::strin return false; } impl_ = std::make_shared(); +#ifdef ONEBIT_NPU + if (forward_encoder(dir)) { + Impl& m = *impl_; + const fs::path d(dir); + // The kernel directory the chat route resolves too: ONEBIT_NPU_KERNELS, else /kernels, else the build's. + std::string kernels; + if (const char* e = std::getenv("ONEBIT_NPU_KERNELS")) kernels = e; + if (kernels.empty() && fs::is_directory(d / "kernels")) kernels = (d / "kernels").string(); +#ifdef ONEBIT_NPU_KERNELS_DIR + if (kernels.empty()) kernels = ONEBIT_NPU_KERNELS_DIR; +#endif + const bool host = device == "cpu" || std::getenv("NPU_INFER_HOST_GEMM") != nullptr; + if (host) setenv("NPU_INFER_HOST_GEMM", "1", 1); // the forward reads it: --device cpu is host GEMM + if (!host && kernels.empty()) { + why = "no NPU kernel (full ELF) directory resolves for this encoder: set ONEBIT_NPU_KERNELS or put them in " + + (d / "kernels").string(); + return false; + } + m.fwd = std::make_unique(); + const std::string model = (d / "model.q4nx").string(); + if (!host) { + if (npu_lock_taken(true, std::getenv("NPU_NO_DEVICE_LOCK") != nullptr)) m.lock_fd = take_npu_device_lock(1); + m.dev = std::make_unique(0); + } + const bool ok = m.dev ? m.fwd->init(*m.dev, model.c_str(), kernels) : m.fwd->init_host(model.c_str(), kernels); + if (!ok) { why = std::string("encoder init failed: ") + m.fwd->last_error(); return false; } + if (!m.fwd->noncausal()) { why = "config.json declares use_bidirectional_attention, but the forward is causal"; return false; } + try { + m.tok = std::make_unique((d / "tokenizer.json").string()); + } catch (const std::exception& e) { + why = std::string("tokenizer.json: ") + e.what(); + return false; + } + // The framing the model was trained with: tokenizer_config.json's add_bos_token / add_eos_token (both true + // on Embedding-Gemma-300M-NPU2, ` text `, the reference tokenizer's ids exactly). + std::ifstream tc(d / "tokenizer_config.json"); + json t = json::object(), c = json::object(); + try { if (tc) t = json::parse(tc); } catch (const std::exception&) {} + std::ifstream cf(d / "config.json"); + try { if (cf) c = json::parse(cf); } catch (const std::exception&) {} + auto token_of = [&](const char* key) { + if (!t.contains(key) || !t[key].is_string()) return -1; + return m.tok->token_id(t[key].get()); + }; + if (t.value("add_bos_token", false)) m.bos = token_of("bos_token"); + if (t.value("add_eos_token", false)) m.eos = token_of("eos_token"); + const int win = c.value("sliding_window", 0); + m.cap = win > 0 ? win / 2 : c.value("max_position_embeddings", 512); + std::vector probe; + if (m.bos >= 0) probe.push_back(m.bos); + if (m.eos >= 0) probe.push_back(m.eos); + if (probe.empty()) probe.push_back(0); + std::vector v; + if (!m.fwd->embed_pooled(probe, v)) { why = std::string("encoder warm-up failed: ") + m.fwd->last_error(); return false; } + device_ = host ? "cpu" : "npu"; + dim_ = (int)v.size(); + id_ = d.filename().string(); + return true; + } +#endif if (!impl_->model.load(dir, why)) return false; device_ = device; dim_ = impl_->model.config().out_dim; @@ -80,7 +172,12 @@ bool Backend::load(const std::string& dir, const std::string& device, std::strin return true; } -int Backend::max_tokens() const { return impl_ ? impl_->model.config().max_tokens : 0; } +int Backend::max_tokens() const { +#ifdef ONEBIT_NPU + if (impl_ && impl_->fwd) return impl_->cap; +#endif + return impl_ ? impl_->model.config().max_tokens : 0; +} std::string Backend::health_json(bool ready) const { return json{{"status", ready ? "ok" : "loading"}, @@ -132,6 +229,27 @@ std::string Backend::embeddings(const std::string& body, int& status) { long tokens = 0; for (size_t i = 0; i < texts.size(); ++i) { std::string why; +#ifdef ONEBIT_NPU + if (impl_->fwd) { + Impl& m = *impl_; + std::vector ids; + if (m.bos >= 0) ids.push_back(m.bos); + for (int t : m.tok->encode(texts[i])) ids.push_back(t); + if (m.eos >= 0) ids.push_back(m.eos); + tokens += static_cast(ids.size()); + if ((int)ids.size() > m.cap) + return fail(400, "input[" + std::to_string(i) + "] is " + std::to_string(ids.size()) + + " tokens; this encoder takes at most " + std::to_string(m.cap)); + std::vector v; + { + std::lock_guard g(m.mu); + if (!m.fwd->embed_pooled(ids, v)) why = m.fwd->last_error(); + } + if (v.empty()) return fail(500, "embedding of input[" + std::to_string(i) + "] failed: " + why); + data.push_back({{"object", "embedding"}, {"index", static_cast(i)}, {"embedding", v}}); + continue; + } +#endif const auto ids = impl_->model.tokenize(texts[i]); tokens += static_cast(ids.size()); const auto v = impl_->model.encode_ids(ids, why); diff --git a/app/embed_serve.h b/app/embed_serve.h index 5aa698a87..cc98b47de 100644 --- a/app/embed_serve.h +++ b/app/embed_serve.h @@ -38,6 +38,11 @@ namespace onebit::embed_serve { // True when `path` is a 1-bit encoder container rather than a GGUF or a q4nx directory. bool container(const std::string& path); +// engine#1786: true when `path` is a q4nx directory whose config.json declares `use_bidirectional_attention: true` +// (FastFlowLM's Embedding-Gemma-300M-NPU2): an encoder the per-op forward runs in one non-causal pass, pooled +// (Q4nxNpuForward::embed_pooled). Served by the same flags and the same /v1/embeddings surface. NPU builds only. +bool forward_encoder(const std::string& path); + class Backend { public: // `device` is "npu" or "cpu". Loading reads the container and the vocabulary; on "npu" the ternary diff --git a/app/serve.cpp b/app/serve.cpp index bf4bffdb2..957da26f0 100644 --- a/app/serve.cpp +++ b/app/serve.cpp @@ -3402,7 +3402,7 @@ int run_serve(int argc, char** argv) { if (o.device == "npu") return serve_only("--device npu runs in serve's own process"); } - if (embed_serve::container(o.model)) { + if (embed_serve::container(o.model) || (o.role == "embedding" && embed_serve::forward_encoder(o.model))) { // `--embedding -m [--device npu|cpu]`: a 1-bit encoder container is served in-process by the // engine's own encoder, so it must be dispatched BEFORE the NPU-directory rules below (those are about // q4nx/forward-serve directories, which do not run on the host). app/embed_serve.h. diff --git a/docs/npu.md b/docs/npu.md index 10efc60ca..5899764da 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -273,6 +273,46 @@ Measured on this box through that route, 1,000 fixed sentences (the gate's own s and STS-B through the same route: **Spearman ×100 82.9412** (the model card's 82.87, its own runtime 82.75 on the same evaluator). +### Sentence embeddings from a q4nx encoder (Embedding-Gemma-300M-NPU2, engine#1786) + +FastFlowLM's `Embedding-Gemma-300M-NPU2` is a `Gemma3TextModel` with `use_bidirectional_attention: true` plus the +sentence-transformers head (`dense_2` 768 -> 3072, `dense_3` 3072 -> 768). The same flags serve it: + +``` +1bit serve --embedding -m --device npu|cpu # ONEBIT_NPU_KERNELS, /kernels or the build's set +``` + +`/v1/embeddings` has the shape described above. The per-op forward encodes the input in one batched pass at +position 0 (`Q4nxNpuForward::embed_pooled`). Every layer reads the whole input in both directions, sliding layers +included. That equals the band a sliding layer was trained with only while the input fits half the window, so an +input longer than 256 tokens (the window is 512) is refused with a 400 naming the cap. The rows are mean-pooled +after the final norm, then pass through `dense_2` and `dense_3` (linear, no bias) and are L2-normalised. Ids are +framed as the pack's `tokenizer_config.json` says (`add_bos_token`, `add_eos_token`: ` text `), which gives +the reference tokenizer's ids exactly. + +Four things about this pack differ from the other gemma3 packs, and each is read from the pack: +- the tensors carry no `model.` prefix (`layers.N.*`, `embed_tokens`, `norm`), and the loader prefixes them; +- the projections are unquantised BF16, read as they are; +- the embedding is stored pre-scaled by sqrt(H), so the forward applies no input scale; +- the norms are HF's raw `w`. FastFlowLM's quantised gemma3 packs store `1 + w`; this one is a copy of the HF + checkpoint, so a gemma3 pack with BF16 projections is run as `1 + w` (the load prints `rmsnorm_offset=1 <- gemma3 + with BF16 projections`). + +Measured against transformers' own `Gemma3TextModel` on `unsloth/embeddinggemma-300m` (the ungated copy of google's +checkpoint, mean pooling and the two dense modules as sentence-transformers applies them), five sentences, the same +ids: + +| route | cos to the reference (min of 5) | 5 inputs | +|---|---|---| +| `--device cpu` (host GEMM) | 0.999990 | | +| `--device npu` | 0.999093 | 0.88 s | + +The similarity matrix is the reference's to three decimals on the host (the two cat sentences 0.885, the unrelated +pairs 0.09 to 0.28). Each rule matters: with the norm rule off the host reads cos 0.09, and with the sliding layers +left causal 0.70 to 0.90. `tests/embed_forward_ci.py` (`embed_forward`) pins the encoder on a fabricated two-layer +container in this layout against a numpy Gemma3 encoder (cos 1.0000000). The same two mutants fail it (0.99 and +0.24). + ### Q4NX chunks Q4NX is [FastFlowLM](https://github.com/ROCm/FastFlowLM)'s NPU model format (MIT), and its models on diff --git a/npu/forward/include/model.h b/npu/forward/include/model.h index 1e06af792..f82164dda 100644 --- a/npu/forward/include/model.h +++ b/npu/forward/include/model.h @@ -140,6 +140,11 @@ TensorDesc* model_tensor_by_name(ModelWeights* mw, const char* name); // form, where post_attention_layernorm is the attention/conv output's norm) and post_attention_layernorm // otherwise. Returns the number of tensors renamed. int model_alias_lfm_names(TensorDesc* tensors, int count); +// engine#1786: FastFlowLM's Embedding-Gemma-300M-NPU2 keeps the sentence-transformers module's names, with no +// `model.` prefix: `layers.N.*`, `embed_tokens.weight`, `norm.weight` (beside its `dense_2` / `dense_3` head). This +// prefixes those three forms with `model.`, in place and only where the prefixed name is not already taken, so the +// loader addresses them as it does every other pack. Returns the number of tensors renamed. +int model_alias_unprefixed_names(TensorDesc* tensors, int count); // engine#1786 (item F1): three FastFlowLM NPU2 model sources spell the per-layer prefix `model.layer.` (singular) // where this forward addresses `model.layers.`. Writes the OTHER spelling into `out` and returns 1 when `name` carries // either prefix, 0 when it does not. Only the leading prefix is swapped, once: a caller that wants a fallback retries diff --git a/npu/forward/include/q4nx_forward.h b/npu/forward/include/q4nx_forward.h index 4c38db9aa..f0262415d 100644 --- a/npu/forward/include/q4nx_forward.h +++ b/npu/forward/include/q4nx_forward.h @@ -224,6 +224,10 @@ class Q4nxNpuForward { // The post-final-norm hidden of EVERY row of the last batched pass (M x H, row-major): a speculative drafter // has to advance its own stream over the rows that were actually accepted, and those are exactly these. const std::vector& last_hidden_rows() const { return last_hidden_rows_; } + // engine#1786: a bidirectional encoder's sentence embedding (EmbeddingGemma: Gemma3TextModel plus the + // sentence-transformers head). One batched pass over `ids` at position 0, the mean of the final-normed rows, + // then the pack's dense_2 and dense_3 when it carries them, then L2. Refused by name for a causal model. + bool embed_pooled(const std::vector& ids, std::vector& out); // engine#1780: the residual stream after the listed layers, for a draft that reads the target's hidden states // (a DFlash draft's target_layer_ids). `layers` must be ascending and inside the model; every pass then appends // one row per position to *sink, [taps * H] wide, the listed layers side by side in the order given -- the @@ -434,6 +438,12 @@ class Q4nxNpuForward { // prompt-encoding forward (step_batch at start_pos 0) attends bidirectionally there and the GDN // layers keep their recurrence. Off unless the plan says so, so a causal model is untouched. bool noncausal_attn_ = false; + // use_bidirectional_attention (an encoder): the sliding layers read the whole prompt too, which the band they + // were trained with equals only while the prompt fits half the window -- a longer one is refused by name. + bool bidir_sliding_ = false; + bool hidden_only_ = false; // embed_pooled(): the batched pass stops after the final norm + std::vector dense2_, dense3_; // sentence-transformers head, f32, loaded on first use + int dense2_out_ = 0, dense3_out_ = 0; // engine#1959: measured wall of one batched launch per design, narrow (the 16-row twin) and wide (the prefill // design), for cached weights only. npu_gemm_rows() reads them to decide whether a 17..wide-row chunk runs as one // wide launch or as several narrow ones (batch_split.h). 0 = not measured yet. `split_mode_` is NPU_PREFILL_SPLIT: diff --git a/npu/forward/include/q4nx_semantics.h b/npu/forward/include/q4nx_semantics.h index a5906aa31..566d63219 100644 --- a/npu/forward/include/q4nx_semantics.h +++ b/npu/forward/include/q4nx_semantics.h @@ -213,6 +213,13 @@ typedef struct { * model is untied (its lm_head.weight is not the embedding), so an absent key is not evidence of a tie: the * forward then decides from the container's own two tables (engine#1786). */ int tie_stated; + /* `use_bidirectional_attention: true` (EmbeddingGemma, Gemma3TextModel): an encoder. Every layer, sliding ones + * included, attends both ways over the prompt; the forward encodes it in one batched pass at position 0 and + * cannot generate (engine#1786). */ + int bidirectional; + /* gemma3: the norm convention follows the writer -- FastFlowLM's quantised packs store 1 + w, an unquantised + * (BF16-projection) copy of the HF checkpoint stores w. Resolved by the forward from layer 0's q_proj. */ + int gemma3_bf16_norm_probe; } Q4nxSemantics; /* Identifier for each flag, for the auditable field mapping. */ diff --git a/npu/forward/src/model.c b/npu/forward/src/model.c index b65b67383..c902da99c 100644 --- a/npu/forward/src/model.c +++ b/npu/forward/src/model.c @@ -854,6 +854,10 @@ ModelWeights* model_load(const char* path, ModelConfig config) { const int renamed = model_alias_lfm_names(tensors, num_tensors); if (renamed > 0) LOG_INFO("Renamed %d LFM tensor names to the canonical ones", renamed); } + { + const int renamed = model_alias_unprefixed_names(tensors, num_tensors); + if (renamed > 0) LOG_INFO("Prefixed %d unprefixed tensor names with model. (sentence-transformers layout)", renamed); + } // engine#1185: a container imported from an OGA pack stores its I8 weights as 6,144-byte q4_1f // chunks (fp32 scale/zero) instead of the 5,120-byte q4_1 the kernels here dequantise in-kernel. @@ -1220,6 +1224,22 @@ int model_alias_lfm_names(TensorDesc* t, int count) { return renamed; } +int model_alias_unprefixed_names(TensorDesc* t, int count) { + if (!t || count <= 0) return 0; + int renamed = 0; + char want[sizeof(t[0].name)]; + for (int i = 0; i < count; i++) { + const char* n = t[i].name; + const int layer = strncmp(n, "layers.", 7) == 0; + if (!layer && strcmp(n, "embed_tokens.weight") != 0 && strcmp(n, "norm.weight") != 0) continue; + if (snprintf(want, sizeof(want), "model.%s", n) >= (int)sizeof(want)) continue; + if (name_taken(t, count, want)) continue; + snprintf(t[i].name, sizeof(t[i].name), "%s", want); + renamed++; + } + return renamed; +} + TensorDesc* model_tensor_by_name(ModelWeights* mw, const char* name) { if (!mw || !mw->all_tensors || !name) return NULL; for (int i = 0; i < mw->all_count; i++) { diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index e0e8e0e6e..a71d3250d 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -199,6 +199,18 @@ static int64_t declared_row_bytes(const TensorDesc* d) { static int dequant_any(const void* data, size_t nbytes, int rows, int cols, float* out, const TensorDesc* desc = nullptr) { if (!data || !out || rows <= 0 || cols <= 0) return -1; + // engine#1786: an unquantised projection (FastFlowLM's Embedding-Gemma-300M-NPU2 ships every weight as BF16 + // [out, in]). Copied into the forward's grid, which may be padded past the declared shape (IM 1152 -> 1280): the + // pad rows and columns are zero, the layout every chunked reader below writes. + if (desc && desc->ndim == 2 && !strcmp(desc->dtype, "BF16")) { + const int64_t r0 = desc->shape[0], c0 = desc->shape[1]; + if (r0 <= 0 || c0 <= 0 || r0 > rows || c0 > cols || nbytes < (size_t)r0 * (size_t)c0 * 2) return -1; + const uint16_t* w = (const uint16_t*)data; + for (int r = 0; r < rows; r++) + for (int c = 0; c < cols; c++) + out[(size_t)r * cols + c] = (r < r0 && c < c0) ? bf16_to_f32(w[(size_t)r * (size_t)c0 + (size_t)c]) : 0.0f; + return 0; + } const int64_t decl_row_bytes = declared_row_bytes(desc); if (decl_row_bytes == Q4NX_CHUNK_D2560) { // GPT-OSS: [RB, CB, S, 2560]; the container's K (CB*128) may be padded past the model's, and is read into @@ -1141,6 +1153,14 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, return false; } if (sem_.eps <= 0) sem_.eps = c_.eps; + if (sem_.bidirectional) { + noncausal_attn_ = true; + bidir_sliding_ = true; + fprintf(stderr, " [forward] use_bidirectional_attention: an encoder -- every layer, sliding ones included, " + "reads the whole prompt in one pass at position 0 (up to %d tokens, half the %d-token " + "window); it cannot generate\n", sem_.sliding_window > 0 ? sem_.sliding_window / 2 : 0, + sem_.sliding_window); + } // engine#1788: whether q and k are normalised per head is a fact about the container, not about its name // (q4nx_semantics.h). A layer that carries both q_norm and k_norm turns it on for a model_type the table // does not know; said out loud, because the dump below then prints 1 beside "<- model_type". @@ -1877,6 +1897,14 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, // official 0.8B release is zero-centered; the hybrid 27B packs are plain gains). Resolve it from the // weight that was just loaded, and print the evidence so a surprising answer is visible rather than // buried. Every other model_type keeps the convention it already decided above. + if (sem_.gemma3_bf16_norm_probe && !sem_.rmsnorm_offset) { + const TensorDesc* q0 = model_tensor_by_name(mw_, "model.layers.0.self_attn.q_proj.weight"); + if (q0 && !strcmp(q0->dtype, "BF16")) { + sem_.rmsnorm_offset = 1; + fprintf(stderr, " [semantics] rmsnorm_offset=1 <- gemma3 with BF16 projections (an unconverted HF copy: its " + "norms are HF's w, applied as 1 + w)\n"); + } + } if (sem_.rmsnorm_offset_probe && !in_norm_.empty() && c_.H > 0) { const std::vector& w0 = in_norm_[0]; double sum = 0.0; @@ -4377,6 +4405,66 @@ bool Q4nxNpuForward::step_batch_all(const std::vector& tokens, int start_po // engine#1780: a pass that fails must not leave rows in the tap sink. The bodies below return false from many // places, so the rule is applied once, here, around each of them. +bool Q4nxNpuForward::embed_pooled(const std::vector& ids, std::vector& out) { + out.clear(); + if (!noncausal_attn_) { + err_ = "embed_pooled(): this model is causal (no use_bidirectional_attention, no noncausal plan); a sentence " + "embedding pools an encoder's rows"; + return false; + } + if (ids.empty()) { err_ = "embed_pooled(): empty input"; return false; } + const int H = c_.H; + // The sentence-transformers head, as stored: dense_2 [D2, H] then dense_3 [D3, D2], both linear (identity + // activation). Read once, f32; a pack without them pools to H. + auto load_dense = [&](const char* name, int in_w, std::vector& w, int& rows) -> bool { + if (!w.empty()) return true; + TensorDesc* d = model_tensor_by_name(mw_, name); + if (!d) { rows = 0; return true; } + if (d->ndim != 2 || d->shape[1] != in_w || strcmp(d->dtype, "BF16") != 0) { + err_ = std::string("embed_pooled(): ") + name + " is not a BF16 [out, " + std::to_string(in_w) + "] matrix"; + return false; + } + rows = (int)d->shape[0]; + const uint16_t* raw = (const uint16_t*)model_tensor_data(mw_, d); + if (!raw || (size_t)d->data_size < (size_t)rows * (size_t)in_w * 2) { err_ = std::string("embed_pooled(): ") + name + " is short"; return false; } + w.resize((size_t)rows * (size_t)in_w); + for (size_t i = 0; i < w.size(); i++) w[i] = bf16_to_f32(raw[i]); + return true; + }; + if (!load_dense("dense_2.weight", H, dense2_, dense2_out_)) return false; + if (dense2_out_ > 0 && !load_dense("dense_3.weight", dense2_out_, dense3_, dense3_out_)) return false; + std::vector unused; + hidden_only_ = true; + const bool ok = step_batch(ids, 0, unused); + hidden_only_ = false; + if (!ok) return false; + const int M = (int)ids.size(); + if (last_hidden_rows_.size() != (size_t)M * (size_t)H) { err_ = "embed_pooled(): row count"; return false; } + std::vector mean((size_t)H, 0.0); + for (int m = 0; m < M; m++) + for (int k = 0; k < H; k++) mean[(size_t)k] += last_hidden_rows_[(size_t)m * H + k]; + std::vector v((size_t)H); + for (int k = 0; k < H; k++) v[(size_t)k] = (float)(mean[(size_t)k] / M); + auto linear = [](const std::vector& w, int rows, int in_w, const std::vector& x) { + std::vector y((size_t)rows); + for (int r = 0; r < rows; r++) { + double s = 0.0; + const float* wr = &w[(size_t)r * (size_t)in_w]; + for (int k = 0; k < in_w; k++) s += (double)wr[k] * x[(size_t)k]; + y[(size_t)r] = (float)s; + } + return y; + }; + if (dense2_out_ > 0) v = linear(dense2_, dense2_out_, H, v); + if (dense3_out_ > 0) v = linear(dense3_, dense3_out_, dense2_out_, v); + double n2 = 0.0; + for (float x : v) n2 += (double)x * x; + const double inv = n2 > 0 ? 1.0 / std::sqrt(n2) : 0.0; + for (float& x : v) x = (float)(x * inv); + out = std::move(v); + return true; +} + bool Q4nxNpuForward::step_batch(const std::vector& tokens, int start_pos, std::vector& logits) { onebit::npu::LayerTapPass pass(tap_sink_); tap_fault_ = false; @@ -4409,6 +4497,14 @@ bool Q4nxNpuForward::step_batch_impl(const std::vector& tokens, int start_p const int H = c_.H; const int M = (int)tokens.size(); if (M <= 0) { err_ = "empty batch"; return false; } + // A bidirectional sliding layer was trained on a band around each position; reading the whole prompt is that + // band only while the prompt fits half the window, whichever side convention the band uses. + if (bidir_sliding_ && sem_.sliding_window > 0 && M > sem_.sliding_window / 2) { + err_ = "use_bidirectional_attention: a " + std::to_string(M) + "-token input is longer than half the " + + std::to_string(sem_.sliding_window) + "-token sliding window, where a sliding layer's band is no " + + "longer the whole prompt; send at most " + std::to_string(sem_.sliding_window / 2) + " tokens"; + return false; + } for (int t : tokens) if (t < 0 || t >= c_.NV) { err_ = "token out of range"; return false; } @@ -4751,7 +4847,7 @@ bool Q4nxNpuForward::step_batch_impl(const std::vector& tokens, int start_p return e && e[0] == '1'; }(); if (attn_rows_on && M > 1 && l_hd > 0 && c_.NH % l_nkv == 0) { - const bool whole = noncausal_attn_ && l_full; + const bool whole = noncausal_attn_ && (l_full || bidir_sliding_); const size_t qw = (size_t)nhd; std::vector qall((size_t)M * qw), qm; std::vector Tm((size_t)M), t0m((size_t)M); @@ -4794,7 +4890,7 @@ bool Q4nxNpuForward::step_batch_impl(const std::vector& tokens, int start_p return false; } } - } else if (noncausal_attn_ && l_full) { + } else if (noncausal_attn_ && (l_full || bidir_sliding_)) { for (int m = 0; m < M; m++) if (!row_attn(m, nullptr, true, 0)) return false; for (int m = 0; m < M; m++) @@ -5007,13 +5103,20 @@ bool Q4nxNpuForward::step_batch_impl(const std::vector& tokens, int start_p logits.assign(batch_all_out_->begin() + (size_t)(M - 1) * (size_t)c_.NV, batch_all_out_->end()); // engine#1781: keep the batched rows so the oracle can compare all of them, not just the last one's argmax. if (batch_check) bat_rows.assign(batch_all_out_->begin(), batch_all_out_->end()); + } else if (hidden_only_) { + // embed_pooled(): an encoder has no next token, so the head is not run; every row is normed and kept. + last_hidden_rows_.assign((size_t)M * (size_t)H, 0.0f); + for (int m = 0; m < M; m++) + rmsnorm(&X[(size_t)m * H], final_norm_.data(), H, &last_hidden_rows_[(size_t)m * H]); + std::copy(last_hidden_rows_.begin() + (size_t)(M - 1) * H, last_hidden_rows_.end(), fnorm.begin()); + logits.clear(); } else { std::vector last(X.begin() + (size_t)(M - 1) * H, X.end()); rmsnorm(last.data(), final_norm_.data(), H, fnorm.data()); ok = run_lm_head(fnorm, logits); } last_hidden_ = fnorm; // for last_hidden(): the same vector the head reads, last position only - if (getenv("NPU_INFER_DUMP_LAYERS")) { + if (getenv("NPU_INFER_DUMP_LAYERS") && !logits.empty()) { int best[5] = {-1, -1, -1, -1, -1}; for (int i = 0; i < (int)logits.size(); i++) { for (int r = 0; r < 5; r++) { diff --git a/npu/forward/src/q4nx_semantics.c b/npu/forward/src/q4nx_semantics.c index acacd0a0e..79500c9ef 100644 --- a/npu/forward/src/q4nx_semantics.c +++ b/npu/forward/src/q4nx_semantics.c @@ -226,6 +226,10 @@ int q4nx_semantics_from_config(const char* json_path, const char* model_dir, /* qwen3_5 ships both conventions under the same model_type, so the forward resolves it from the * layer-0 input_layernorm weight it loads (a config-only answer would be a guess). */ s->rmsnorm_offset_probe = (is_qwen35 && !is_gemma) ? 1 : 0; + /* engine#1786: a gemma3 pack whose projections are unquantised BF16 is a copy of the HF checkpoint, norms + * included (Embedding-Gemma-300M-NPU2: every norm equals unsloth/embeddinggemma-300m's to bf16 rounding), so it + * carries HF's raw w, not the converted 1 + w. The forward decides from layer 0's q_proj dtype. */ + s->gemma3_bf16_norm_probe = is_gemma3 ? 1 : 0; /* LFM2 / LFM2.5 (Lfm2ForCausalLM): its attention layers carry q_norm/k_norm, applied before RoPE */ const int is_lfm2 = starts_with(model_type, "lfm2") || starts_with(arch, "Lfm2"); @@ -432,6 +436,8 @@ int q4nx_semantics_from_config(const char* json_path, const char* model_dir, * lm_head.weight (Nanbeige4.1-3B sets tie_word_embeddings = false). */ s->tie_embeddings = 1; s->tie_stated = 0; + s->bidirectional = 0; + if (get_num(json, "use_bidirectional_attention", &d)) s->bidirectional = d != 0; if (get_num(json, "tie_word_embeddings", &d)) { s->tie_embeddings = d != 0; s->tie_stated = 1; } /* layer_types -> the per-layer attention kind the forward dispatches on */ diff --git a/tests/embed_forward_ci.py b/tests/embed_forward_ci.py new file mode 100644 index 000000000..3d531d50c --- /dev/null +++ b/tests/embed_forward_ci.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""engine#1786: the bidirectional gemma3 encoder (FastFlowLM's Embedding-Gemma-300M-NPU2) through the per-op forward. + +usage: tests/embed_forward_ci.py (needs numpy) + +Writes a two-layer container in that pack's own layout -- every projection BF16 [out, in], the sentence-transformers +names with no `model.` prefix (`layers.N.*`, `embed_tokens`, `norm`, `dense_2`, `dense_3`), the embedding stored +pre-scaled by sqrt(H), norms as HF's raw w, `use_bidirectional_attention: true`, a sliding layer then a full one, and an +intermediate size off the 256 grid (384 -> 512) -- and holds Q4nxNpuForward::embed_pooled() to a numpy Gemma3 encoder +on the same weights: (1 + w) RMSNorm everywhere, q/k norms, rotate-half RoPE on the whole head (base 10000 on the +sliding layer, rope_theta on the full one), query_pre_attn_scalar, attention over EVERY position, the sandwich norms, +GELU(tanh) gated MLP, mean pooling, dense_2, dense_3, L2. The real pack against transformers' Gemma3TextModel scored +cos 0.99999 on the host; any one of those rules wrong moves this one far off (the norm rule alone: 0.09 on the real +pack, the sliding layers left causal: 0.70 to 0.90). A second request one token past half the window must be refused. +""" +import json +import os +import struct +import subprocess +import sys +import tempfile + +H, NH, NKV, HD, IM, V, D2 = 256, 2, 1, 128, 384, 96, 512 +WIN = 16 # sliding window: inputs of up to WIN // 2 tokens +QPAS = 128.0 # query_pre_attn_scalar +THETA, LOCAL = 1000000.0, 10000.0 +EPS = 1e-6 +MIN_COS = 0.99999 + + +def tmp_root(): + for d in (os.environ.get("ONEBIT_SCRATCH"), os.path.expanduser("~/.cache"), None): + if d is None or os.path.isdir(d): + return d + return None + + +def bf16(np, a): + """Round to bf16 (nearest even) and return (the raw uint16 words, the float32 values they stand for).""" + u = np.ascontiguousarray(a, dtype=np.float32).view(np.uint32) + r = ((u + 0x7FFF + ((u >> 16) & 1)) >> 16).astype(np.uint16) + return r, (r.astype(np.uint32) << 16).view(np.float32) + + +def write_container(path, arrays): + header, blobs, off = {}, [], 0 + for name, raw in arrays.items(): + b = raw.tobytes() + header[name] = {"dtype": "BF16", "shape": list(raw.shape), "data_offsets": [off, off + len(b)]} + blobs.append(b) + off += len(b) + h = json.dumps(header, separators=(",", ":")).encode() + h += b" " * ((-len(h)) % 8) + with open(path, "wb") as f: + f.write(struct.pack("hts", q, k) / np.sqrt(QPAS) # no mask: every position, both ways + sc = np.exp(sc - sc.max(-1, keepdims=True)) + sc /= sc.sum(-1, keepdims=True) + o = np.einsum("hts,shd->thd", sc, v).reshape(T, NH * HD) @ W[p + "self_attn.o_proj.weight"].T + x = x + rms(o, W[p + "post_attention_layernorm.weight"]) + h = rms(x, W[p + "pre_feedforward_layernorm.weight"]) + f = (gelu(h @ W[p + "mlp.gate_proj.weight"].T) * (h @ W[p + "mlp.up_proj.weight"].T)) @ W[p + "mlp.down_proj.weight"].T + x = x + rms(f, W[p + "post_feedforward_layernorm.weight"]) + pooled = rms(x, W["norm.weight"]).mean(0) + e = pooled @ W["dense_2.weight"].T @ W["dense_3.weight"].T + return e / np.linalg.norm(e) + + +def run(binary, d, ids, out): + env = dict(os.environ, ONEBIT_TEST_Q4NX=d, ONEBIT_TEST_IDS=" ".join(map(str, ids)), ONEBIT_TEST_EMBED_OUT=out) + env.pop("NPU_INFER_HOST_GEMM", None) + return subprocess.run([binary], env=env, capture_output=True, text=True, timeout=110) + + +def main(): + if len(sys.argv) != 2: + print(__doc__.strip().splitlines()[2]) + return 2 + try: + import numpy as np + except ImportError: + print("SKIP: numpy is not installed") + return 77 + with tempfile.TemporaryDirectory(prefix="embed_forward_ci.", dir=tmp_root()) as tmp: + d, out = os.path.join(tmp, "enc"), os.path.join(tmp, "vec.f32") + W = make(d, np) + ids = [2, 17, 40, 5, 63, 1] + r = run(sys.argv[1], d, ids, out) + if r.returncode != 0: + print(f"FAIL embed_pooled_test rc={r.returncode}: {(r.stdout + r.stderr)[-800:]}") + return 1 + ours = np.fromfile(out, dtype=np.float32) + ref = reference(np, W, ids) + cos = float(ours @ ref) if ours.shape == ref.shape else float("nan") + ok = ours.shape == ref.shape and cos >= MIN_COS + print(f"{'PASS' if ok else 'FAIL'} embed_pooled vs the numpy bidirectional gemma3 encoder: cos {cos:.7f} " + f"(needs {MIN_COS}), {ours.size} floats, |v| {float(np.linalg.norm(ours)):.6f}") + too_long = list(range(2, 2 + WIN // 2 + 1)) + r2 = run(sys.argv[1], d, too_long, out) + refused = r2.returncode == 3 and "half the" in r2.stdout + print(f"{'PASS' if refused else 'FAIL'} a {len(too_long)}-token input (window {WIN}) is refused by name: " + f"{r2.stdout.strip()[:160]}") + return 0 if ok and refused else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/embed_pooled_test.cpp b/tests/embed_pooled_test.cpp new file mode 100644 index 000000000..0f5b4120e --- /dev/null +++ b/tests/embed_pooled_test.cpp @@ -0,0 +1,61 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// engine#1786: Q4nxNpuForward::embed_pooled() on a container named by ONEBIT_TEST_Q4NX, host GEMM, no device. +// tests/embed_forward_ci.py writes the container (a bidirectional gemma3 encoder in FastFlowLM's +// Embedding-Gemma-300M-NPU2 layout) and holds the vector this writes to ONEBIT_TEST_EMBED_OUT to a numpy reference. +// ONEBIT_TEST_IDS is the space-separated input. Exit 0 on a vector, 3 when the forward refused (its reason on +// stdout, so the script can check a refusal it asked for), 77 when the container is not set. +#include "file_create.h" +#include "q4nx_forward.h" + +#include +#include +#include +#include +#include + +int main() { + const char* dir = std::getenv("ONEBIT_TEST_Q4NX"); + const char* ids_env = std::getenv("ONEBIT_TEST_IDS"); + const char* out_path = std::getenv("ONEBIT_TEST_EMBED_OUT"); + if (!dir || !*dir || !ids_env || !out_path) { + std::printf("SKIP: set ONEBIT_TEST_Q4NX, ONEBIT_TEST_IDS and ONEBIT_TEST_EMBED_OUT\n"); + return 77; + } + setenv("NPU_INFER_HOST_GEMM", "1", 1); + const std::string model = std::string(dir) + "/model.q4nx"; + Q4nxNpuForward fwd; + if (!fwd.init_host(model.c_str(), std::string(dir) + "/kernels")) { + std::printf("FAIL: init_host: %s\n", fwd.last_error()); + return 1; + } + std::vector ids; + std::istringstream is(ids_env); + for (int t; is >> t;) ids.push_back(t); + std::vector v; + if (!fwd.embed_pooled(ids, v)) { + std::printf("REFUSED: %s\n", fwd.last_error()); + return 3; + } + FILE* f = onebit_fopen_private_wb(out_path); // 0600, no symlink follow (file_create.h) + if (!f || std::fwrite(v.data(), sizeof(float), v.size(), f) != v.size()) { + std::printf("FAIL: cannot write %s\n", out_path); + return 1; + } + std::fclose(f); + std::printf("embedded %zu ids -> %zu floats\n", ids.size(), v.size()); + return 0; +}