Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2256,6 +2256,15 @@ if(ONEBIT_NPU_HOST AND NOT ONEBIT_NPU AND NOT WIN32 AND Python3_Interpreter_FOUN
COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/batch_check_state_ci.py
$<TARGET_FILE:batch_check_state_test> qwen3_5)
set_tests_properties(batch_check_state_qwen35 PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 120)
# engine#1786: the bidirectional gemma3 encoder (Embedding-Gemma-300M-NPU2's layout: BF16 projections, unprefixed
# names, pre-scaled embedding, raw norms, a sliding and a full layer) through embed_pooled(), against a numpy
# encoder on the same weights, and an over-long input refused by name.
add_executable(embed_pooled_test tests/embed_pooled_test.cpp)
target_link_libraries(embed_pooled_test PRIVATE onebit_npu_forward_ci)
add_dependencies(onebit embed_pooled_test)
add_test(NAME embed_forward
COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/embed_forward_ci.py $<TARGET_FILE:embed_pooled_test>)
set_tests_properties(embed_forward PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 120)
endif()

# Private NPU routes (docs/npu.md, "Private routes"): closed-source add-ons in a checkout of
Expand Down
120 changes: 119 additions & 1 deletion app/embed_serve.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,13 @@

#ifdef ONEBIT_NPU
#include "embed_npu.h"
#include "../npu/forward/include/q4nx_forward.h"
#include "npu_device_lock.h"
#include "tokenizer.h"
#include "rt.h"
#include <cstdlib>
#include <fstream>
#include <mutex>
#endif

namespace fs = std::filesystem;
Expand All @@ -41,11 +48,36 @@ bool container(const std::string& path) {
return fs::is_directory(path, ec) && fs::is_regular_file(fs::path(path) / "model.embed", ec);
}

bool forward_encoder(const std::string& path) {
#ifdef ONEBIT_NPU
std::error_code ec;
const fs::path d(path);
if (!fs::is_directory(d, ec) || !fs::is_regular_file(d / "model.q4nx", ec)) return false;
std::ifstream f(d / "config.json");
if (!f) return false;
try {
const json c = json::parse(f);
return c.value("use_bidirectional_attention", false);
} catch (const std::exception&) {
return false;
}
#else
(void)path;
return false;
#endif
}

struct Backend::Impl {
embed::Model model;
#ifdef ONEBIT_NPU
embed::NpuGemm npu;
bool npu_installed = false;
// engine#1786: the q4nx encoder kind. One forward, so requests are serialised.
std::unique_ptr<xrt::device> dev;
std::unique_ptr<Q4nxNpuForward> fwd;
std::unique_ptr<onebit::npu::Tokenizer> tok;
int bos = -1, eos = -1, cap = 0, lock_fd = -1;
std::mutex mu;
#endif
};

Expand All @@ -55,6 +87,66 @@ bool Backend::load(const std::string& dir, const std::string& device, std::strin
return false;
}
impl_ = std::make_shared<Impl>();
#ifdef ONEBIT_NPU
if (forward_encoder(dir)) {
Impl& m = *impl_;
const fs::path d(dir);
// The kernel directory the chat route resolves too: ONEBIT_NPU_KERNELS, else <dir>/kernels, else the build's.
std::string kernels;
if (const char* e = std::getenv("ONEBIT_NPU_KERNELS")) kernels = e;
if (kernels.empty() && fs::is_directory(d / "kernels")) kernels = (d / "kernels").string();
#ifdef ONEBIT_NPU_KERNELS_DIR
if (kernels.empty()) kernels = ONEBIT_NPU_KERNELS_DIR;
#endif
const bool host = device == "cpu" || std::getenv("NPU_INFER_HOST_GEMM") != nullptr;
if (host) setenv("NPU_INFER_HOST_GEMM", "1", 1); // the forward reads it: --device cpu is host GEMM
if (!host && kernels.empty()) {
why = "no NPU kernel (full ELF) directory resolves for this encoder: set ONEBIT_NPU_KERNELS or put them in " +
(d / "kernels").string();
return false;
}
m.fwd = std::make_unique<Q4nxNpuForward>();
const std::string model = (d / "model.q4nx").string();
if (!host) {
if (npu_lock_taken(true, std::getenv("NPU_NO_DEVICE_LOCK") != nullptr)) m.lock_fd = take_npu_device_lock(1);
m.dev = std::make_unique<xrt::device>(0);
}
const bool ok = m.dev ? m.fwd->init(*m.dev, model.c_str(), kernels) : m.fwd->init_host(model.c_str(), kernels);
if (!ok) { why = std::string("encoder init failed: ") + m.fwd->last_error(); return false; }
if (!m.fwd->noncausal()) { why = "config.json declares use_bidirectional_attention, but the forward is causal"; return false; }
try {
m.tok = std::make_unique<onebit::npu::Tokenizer>((d / "tokenizer.json").string());
} catch (const std::exception& e) {
why = std::string("tokenizer.json: ") + e.what();
return false;
}
// The framing the model was trained with: tokenizer_config.json's add_bos_token / add_eos_token (both true
// on Embedding-Gemma-300M-NPU2, `<bos> text <eos>`, the reference tokenizer's ids exactly).
std::ifstream tc(d / "tokenizer_config.json");
json t = json::object(), c = json::object();
try { if (tc) t = json::parse(tc); } catch (const std::exception&) {}
std::ifstream cf(d / "config.json");
try { if (cf) c = json::parse(cf); } catch (const std::exception&) {}
auto token_of = [&](const char* key) {
if (!t.contains(key) || !t[key].is_string()) return -1;
return m.tok->token_id(t[key].get<std::string>());
};
if (t.value("add_bos_token", false)) m.bos = token_of("bos_token");
if (t.value("add_eos_token", false)) m.eos = token_of("eos_token");
const int win = c.value("sliding_window", 0);
m.cap = win > 0 ? win / 2 : c.value("max_position_embeddings", 512);
std::vector<int> probe;
if (m.bos >= 0) probe.push_back(m.bos);
if (m.eos >= 0) probe.push_back(m.eos);
if (probe.empty()) probe.push_back(0);
std::vector<float> v;
if (!m.fwd->embed_pooled(probe, v)) { why = std::string("encoder warm-up failed: ") + m.fwd->last_error(); return false; }
device_ = host ? "cpu" : "npu";
dim_ = (int)v.size();
id_ = d.filename().string();
return true;
}
#endif
if (!impl_->model.load(dir, why)) return false;
device_ = device;
dim_ = impl_->model.config().out_dim;
Expand All @@ -80,7 +172,12 @@ bool Backend::load(const std::string& dir, const std::string& device, std::strin
return true;
}

int Backend::max_tokens() const { return impl_ ? impl_->model.config().max_tokens : 0; }
int Backend::max_tokens() const {
#ifdef ONEBIT_NPU
if (impl_ && impl_->fwd) return impl_->cap;
#endif
return impl_ ? impl_->model.config().max_tokens : 0;
}

std::string Backend::health_json(bool ready) const {
return json{{"status", ready ? "ok" : "loading"},
Expand Down Expand Up @@ -132,6 +229,27 @@ std::string Backend::embeddings(const std::string& body, int& status) {
long tokens = 0;
for (size_t i = 0; i < texts.size(); ++i) {
std::string why;
#ifdef ONEBIT_NPU
if (impl_->fwd) {
Impl& m = *impl_;
std::vector<int> ids;
if (m.bos >= 0) ids.push_back(m.bos);
for (int t : m.tok->encode(texts[i])) ids.push_back(t);
if (m.eos >= 0) ids.push_back(m.eos);
tokens += static_cast<long>(ids.size());
if ((int)ids.size() > m.cap)
return fail(400, "input[" + std::to_string(i) + "] is " + std::to_string(ids.size()) +
" tokens; this encoder takes at most " + std::to_string(m.cap));
std::vector<float> v;
{
std::lock_guard<std::mutex> g(m.mu);
if (!m.fwd->embed_pooled(ids, v)) why = m.fwd->last_error();
}
if (v.empty()) return fail(500, "embedding of input[" + std::to_string(i) + "] failed: " + why);
data.push_back({{"object", "embedding"}, {"index", static_cast<int>(i)}, {"embedding", v}});
continue;
}
#endif
const auto ids = impl_->model.tokenize(texts[i]);
tokens += static_cast<long>(ids.size());
const auto v = impl_->model.encode_ids(ids, why);
Expand Down
5 changes: 5 additions & 0 deletions app/embed_serve.h
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,11 @@ namespace onebit::embed_serve {
// True when `path` is a 1-bit encoder container rather than a GGUF or a q4nx directory.
bool container(const std::string& path);

// engine#1786: true when `path` is a q4nx directory whose config.json declares `use_bidirectional_attention: true`
// (FastFlowLM's Embedding-Gemma-300M-NPU2): an encoder the per-op forward runs in one non-causal pass, pooled
// (Q4nxNpuForward::embed_pooled). Served by the same flags and the same /v1/embeddings surface. NPU builds only.
bool forward_encoder(const std::string& path);

class Backend {
public:
// `device` is "npu" or "cpu". Loading reads the container and the vocabulary; on "npu" the ternary
Expand Down
2 changes: 1 addition & 1 deletion app/serve.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3402,7 +3402,7 @@ int run_serve(int argc, char** argv) {
if (o.device == "npu") return serve_only("--device npu runs in serve's own process");
}

if (embed_serve::container(o.model)) {
if (embed_serve::container(o.model) || (o.role == "embedding" && embed_serve::forward_encoder(o.model))) {
// `--embedding -m <container> [--device npu|cpu]`: a 1-bit encoder container is served in-process by the
// engine's own encoder, so it must be dispatched BEFORE the NPU-directory rules below (those are about
// q4nx/forward-serve directories, which do not run on the host). app/embed_serve.h.
Expand Down
40 changes: 40 additions & 0 deletions docs/npu.md
Original file line number Diff line number Diff line change
Expand Up @@ -273,6 +273,46 @@ Measured on this box through that route, 1,000 fixed sentences (the gate's own s
and STS-B through the same route: **Spearman ×100 82.9412** (the model card's 82.87, its own runtime 82.75 on the
same evaluator).

### Sentence embeddings from a q4nx encoder (Embedding-Gemma-300M-NPU2, engine#1786)

FastFlowLM's `Embedding-Gemma-300M-NPU2` is a `Gemma3TextModel` with `use_bidirectional_attention: true` plus the
sentence-transformers head (`dense_2` 768 -> 3072, `dense_3` 3072 -> 768). The same flags serve it:

```
1bit serve --embedding -m <pack dir> --device npu|cpu # ONEBIT_NPU_KERNELS, <dir>/kernels or the build's set
```

`/v1/embeddings` has the shape described above. The per-op forward encodes the input in one batched pass at
position 0 (`Q4nxNpuForward::embed_pooled`). Every layer reads the whole input in both directions, sliding layers
included. That equals the band a sliding layer was trained with only while the input fits half the window, so an
input longer than 256 tokens (the window is 512) is refused with a 400 naming the cap. The rows are mean-pooled
after the final norm, then pass through `dense_2` and `dense_3` (linear, no bias) and are L2-normalised. Ids are
framed as the pack's `tokenizer_config.json` says (`add_bos_token`, `add_eos_token`: `<bos> text <eos>`), which gives
the reference tokenizer's ids exactly.

Four things about this pack differ from the other gemma3 packs, and each is read from the pack:
- the tensors carry no `model.` prefix (`layers.N.*`, `embed_tokens`, `norm`), and the loader prefixes them;
- the projections are unquantised BF16, read as they are;
- the embedding is stored pre-scaled by sqrt(H), so the forward applies no input scale;
- the norms are HF's raw `w`. FastFlowLM's quantised gemma3 packs store `1 + w`; this one is a copy of the HF
checkpoint, so a gemma3 pack with BF16 projections is run as `1 + w` (the load prints `rmsnorm_offset=1 <- gemma3
with BF16 projections`).

Measured against transformers' own `Gemma3TextModel` on `unsloth/embeddinggemma-300m` (the ungated copy of google's
checkpoint, mean pooling and the two dense modules as sentence-transformers applies them), five sentences, the same
ids:

| route | cos to the reference (min of 5) | 5 inputs |
|---|---|---|
| `--device cpu` (host GEMM) | 0.999990 | |
| `--device npu` | 0.999093 | 0.88 s |

The similarity matrix is the reference's to three decimals on the host (the two cat sentences 0.885, the unrelated
pairs 0.09 to 0.28). Each rule matters: with the norm rule off the host reads cos 0.09, and with the sliding layers
left causal 0.70 to 0.90. `tests/embed_forward_ci.py` (`embed_forward`) pins the encoder on a fabricated two-layer
container in this layout against a numpy Gemma3 encoder (cos 1.0000000). The same two mutants fail it (0.99 and
0.24).

### Q4NX chunks

Q4NX is [FastFlowLM](https://github.com/ROCm/FastFlowLM)'s NPU model format (MIT), and its models on
Expand Down
5 changes: 5 additions & 0 deletions npu/forward/include/model.h
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,11 @@ TensorDesc* model_tensor_by_name(ModelWeights* mw, const char* name);
// form, where post_attention_layernorm is the attention/conv output's norm) and post_attention_layernorm
// otherwise. Returns the number of tensors renamed.
int model_alias_lfm_names(TensorDesc* tensors, int count);
// engine#1786: FastFlowLM's Embedding-Gemma-300M-NPU2 keeps the sentence-transformers module's names, with no
// `model.` prefix: `layers.N.*`, `embed_tokens.weight`, `norm.weight` (beside its `dense_2` / `dense_3` head). This
// prefixes those three forms with `model.`, in place and only where the prefixed name is not already taken, so the
// loader addresses them as it does every other pack. Returns the number of tensors renamed.
int model_alias_unprefixed_names(TensorDesc* tensors, int count);
// engine#1786 (item F1): three FastFlowLM NPU2 model sources spell the per-layer prefix `model.layer.` (singular)
// where this forward addresses `model.layers.`. Writes the OTHER spelling into `out` and returns 1 when `name` carries
// either prefix, 0 when it does not. Only the leading prefix is swapped, once: a caller that wants a fallback retries
Expand Down
10 changes: 10 additions & 0 deletions npu/forward/include/q4nx_forward.h
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,10 @@ class Q4nxNpuForward {
// The post-final-norm hidden of EVERY row of the last batched pass (M x H, row-major): a speculative drafter
// has to advance its own stream over the rows that were actually accepted, and those are exactly these.
const std::vector<float>& last_hidden_rows() const { return last_hidden_rows_; }
// engine#1786: a bidirectional encoder's sentence embedding (EmbeddingGemma: Gemma3TextModel plus the
// sentence-transformers head). One batched pass over `ids` at position 0, the mean of the final-normed rows,
// then the pack's dense_2 and dense_3 when it carries them, then L2. Refused by name for a causal model.
bool embed_pooled(const std::vector<int>& ids, std::vector<float>& out);
// engine#1780: the residual stream after the listed layers, for a draft that reads the target's hidden states
// (a DFlash draft's target_layer_ids). `layers` must be ascending and inside the model; every pass then appends
// one row per position to *sink, [taps * H] wide, the listed layers side by side in the order given -- the
Expand Down Expand Up @@ -434,6 +438,12 @@ class Q4nxNpuForward {
// prompt-encoding forward (step_batch at start_pos 0) attends bidirectionally there and the GDN
// layers keep their recurrence. Off unless the plan says so, so a causal model is untouched.
bool noncausal_attn_ = false;
// use_bidirectional_attention (an encoder): the sliding layers read the whole prompt too, which the band they
// were trained with equals only while the prompt fits half the window -- a longer one is refused by name.
bool bidir_sliding_ = false;
bool hidden_only_ = false; // embed_pooled(): the batched pass stops after the final norm
std::vector<float> dense2_, dense3_; // sentence-transformers head, f32, loaded on first use
int dense2_out_ = 0, dense3_out_ = 0;
// engine#1959: measured wall of one batched launch per design, narrow (the 16-row twin) and wide (the prefill
// design), for cached weights only. npu_gemm_rows() reads them to decide whether a 17..wide-row chunk runs as one
// wide launch or as several narrow ones (batch_split.h). 0 = not measured yet. `split_mode_` is NPU_PREFILL_SPLIT:
Expand Down
7 changes: 7 additions & 0 deletions npu/forward/include/q4nx_semantics.h
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,13 @@ typedef struct {
* model is untied (its lm_head.weight is not the embedding), so an absent key is not evidence of a tie: the
* forward then decides from the container's own two tables (engine#1786). */
int tie_stated;
/* `use_bidirectional_attention: true` (EmbeddingGemma, Gemma3TextModel): an encoder. Every layer, sliding ones
* included, attends both ways over the prompt; the forward encodes it in one batched pass at position 0 and
* cannot generate (engine#1786). */
int bidirectional;
/* gemma3: the norm convention follows the writer -- FastFlowLM's quantised packs store 1 + w, an unquantised
* (BF16-projection) copy of the HF checkpoint stores w. Resolved by the forward from layer 0's q_proj. */
int gemma3_bf16_norm_probe;
} Q4nxSemantics;

/* Identifier for each flag, for the auditable field mapping. */
Expand Down
20 changes: 20 additions & 0 deletions npu/forward/src/model.c
Original file line number Diff line number Diff line change
Expand Up @@ -854,6 +854,10 @@ ModelWeights* model_load(const char* path, ModelConfig config) {
const int renamed = model_alias_lfm_names(tensors, num_tensors);
if (renamed > 0) LOG_INFO("Renamed %d LFM tensor names to the canonical ones", renamed);
}
{
const int renamed = model_alias_unprefixed_names(tensors, num_tensors);
if (renamed > 0) LOG_INFO("Prefixed %d unprefixed tensor names with model. (sentence-transformers layout)", renamed);
}

// engine#1185: a container imported from an OGA pack stores its I8 weights as 6,144-byte q4_1f
// chunks (fp32 scale/zero) instead of the 5,120-byte q4_1 the kernels here dequantise in-kernel.
Expand Down Expand Up @@ -1220,6 +1224,22 @@ int model_alias_lfm_names(TensorDesc* t, int count) {
return renamed;
}

int model_alias_unprefixed_names(TensorDesc* t, int count) {
if (!t || count <= 0) return 0;
int renamed = 0;
char want[sizeof(t[0].name)];
for (int i = 0; i < count; i++) {
const char* n = t[i].name;
const int layer = strncmp(n, "layers.", 7) == 0;
if (!layer && strcmp(n, "embed_tokens.weight") != 0 && strcmp(n, "norm.weight") != 0) continue;
if (snprintf(want, sizeof(want), "model.%s", n) >= (int)sizeof(want)) continue;
if (name_taken(t, count, want)) continue;
snprintf(t[i].name, sizeof(t[i].name), "%s", want);
renamed++;
}
return renamed;
}

TensorDesc* model_tensor_by_name(ModelWeights* mw, const char* name) {
if (!mw || !mw->all_tensors || !name) return NULL;
for (int i = 0; i < mw->all_count; i++) {
Expand Down
Loading
Loading