Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2149,6 +2149,11 @@ if(ONEBIT_NPU_HOST AND NOT ONEBIT_NPU AND NOT WIN32 AND Python3_Interpreter_FOUN
COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/batch_check_state_ci.py
$<TARGET_FILE:batch_check_state_test> lfm3)
set_tests_properties(batch_check_state_lfm3 PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 120)
# Qwen2 (engine#1786): q/k/v biases with no attention_bias key in config.json, against a numpy Qwen2 reference.
add_test(NAME batch_check_state_qwen2
COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/batch_check_state_ci.py
$<TARGET_FILE:batch_check_state_test> qwen2)
set_tests_properties(batch_check_state_qwen2 PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 120)
# A fabricated qwen3_5 hybrid (a linear-attention layer, then attention), off the kernel's N grid on purpose.
add_test(NAME batch_check_state_qwen35
COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/batch_check_state_ci.py
Expand Down
12 changes: 12 additions & 0 deletions docs/npu.md
Original file line number Diff line number Diff line change
Expand Up @@ -2029,6 +2029,18 @@ off (the template's `<think></think>`), gives `Paris</think>Paris<|im_end|>`. Th
choices: `</think>` after the first `Paris`, then `Paris` again. Asked why the sky is blue, it explains Rayleigh
scattering in correct sentences (80 tokens, greedy). No kernel set for its shapes was built or run (the NPU is off).

**Qwen2's attention biases are read from the container (engine#1786).** The forward adds the q, k and v projection
biases only when `attention_bias` is on. A config that states the key decides it. Qwen2's `config.json` never states
it, though every Qwen2 layer has the three biases. With the key absent the forward loaded the biases and never added
them, so FastFlowLM's `Qwen2.5-3B-Instruct-NPU2` answered the known-answer prompt with noise
(`UNS �leon�leon�ucz …`). The rule is now the one q/k normalisation follows: with the key absent, a layer that carries
a q, k or v bias turns the biases on, and the startup line says so (`attention_bias=1 <- the container`). A config
that says `attention_bias: false` while the container carries biases is refused by name rather than guessed.
After: `Paris<|im_end|>` on the host. `batch_check_state_qwen2` fabricates a Qwen2 with biases and no key and
compares the forward's logits with a numpy Qwen2 forward on the container's own weights: corr 1.0000000, max |d|
6.6e-06 of 9.6. The binary without the fix fails it (corr 0.444, another argmax), and batched == sequential passed
on that binary all the same.

**Per-head q/k normalisation is read from the container (engine#1788).** Whether the forward normalises q
and k per head used to be decided by `config.json`'s `model_type` alone: on for `qwen3*` and `gemma*`.
A pack of the same architecture under another name got it wrong in silence. FastFlowLM's MiniCPM-V 4.7
Expand Down
8 changes: 8 additions & 0 deletions npu/forward/include/q4nx_semantics.h
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,10 @@ typedef struct {
/* Linear biases on q/k/v/o and on the MLP projections.
* Source: attention_bias / mlp_bias (presence of corresponding tensors). */
int attention_bias;
/* 1 when config.json states attention_bias (true or false); 0 when the key is absent, which is Qwen2's case:
* its configs never state it and every layer carries q/k/v biases. q4nx_semantics_attention_bias_from_tensors
* decides the absent case from the container. */
int attention_bias_stated;
int mlp_bias;
/* Multiplier applied to token embeddings before layer 0.
* Source: model_type (gemma* uses sqrt(hidden_size)); else 1.0. */
Expand Down Expand Up @@ -278,6 +282,10 @@ int q4nx_rmsnorm_offset_from_norm(const float* w, int n);
* nothing, when qk_norm is already on, when no layer carries both, or on null/empty input. It never turns the norm
* off: a family the table knows keeps its answer whatever the tensors are. Exported so the rule is testable. */
int q4nx_semantics_qk_norm_from_tensors(Q4nxSemantics* s, const unsigned char* has_q, const unsigned char* has_k, int layers);
/* engine#1786: config.json did not state attention_bias and a layer carries a q, k or v projection bias -> the
* biases are applied (attention_bias = 1); returns that layer, else -1. A config that states attention_bias=false
* while the container carries biases is a disagreement the caller refuses: returns -2. */
int q4nx_semantics_attention_bias_from_tensors(Q4nxSemantics* s, const unsigned char* has_bias, int layers);

/* engine#1889's refusal, as a rule: with q/k norm on, a layer whose q gain is absent would normalise q to zero, so the
* container is refused. That is a statement about layers with SOFTMAX ATTENTION. A layer of another kind carries no
Expand Down
18 changes: 18 additions & 0 deletions npu/forward/src/q4nx_forward.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1099,6 +1099,24 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path,
fprintf(stderr, " [semantics] qk_norm=1 <- the container: layer %d carries self_attn.q_norm and k_norm "
"(config.json's model_type is not a family the table knows)\n", by);
}
// engine#1786: attention_bias the same way. Qwen2's config.json never states it and every layer carries q/k/v
// biases; with the key absent they were loaded and never added, so Qwen2.5-3B-Instruct-NPU2 answered noise.
if (mw_ && mw_->config.num_layers > 0) {
std::vector<unsigned char> has_b(size_t(mw_->config.num_layers), 0);
auto present = [](const TensorDesc& t) { return t.name[0] && t.data_size > 0; };
for (int l = 0; l < mw_->config.num_layers; l++)
has_b[size_t(l)] = present(mw_->layers[l].q_proj_bias) || present(mw_->layers[l].k_proj_bias) ||
present(mw_->layers[l].v_proj_bias);
const int by = q4nx_semantics_attention_bias_from_tensors(&sem_, has_b.data(), mw_->config.num_layers);
if (by == -2) {
err_ = "config.json says attention_bias is false and model.q4nx carries q/k/v projection biases; "
"serving it either way would be a guess";
return false;
}
if (by >= 0)
fprintf(stderr, " [semantics] attention_bias=1 <- the container: layer %d carries q/k/v projection "
"biases (config.json does not state attention_bias)\n", by);
}
q4nx_semantics_dump(&sem_);
// A layer kind nothing implements is refused BY NAME, here beside the other precise refusals (engine#1087's
// shape): q4nx_semantics maps an unrecognised layer_types entry to Q4NX_LAYER_OTHER and the attention path
Expand Down
13 changes: 12 additions & 1 deletion npu/forward/src/q4nx_semantics.c
Original file line number Diff line number Diff line change
Expand Up @@ -478,7 +478,7 @@ int q4nx_semantics_from_config(const char* json_path, const char* model_dir,
/* biases: absent field means "no bias" for llama-family configs, but Phi
* and Gemma configs omit it as well, so default to 0 and let phase 2's
* tensor-level probe (bias tensors present?) override it. */
if (get_num(json, "attention_bias", &d)) s->attention_bias = d != 0;
if (get_num(json, "attention_bias", &d)) { s->attention_bias = d != 0; s->attention_bias_stated = 1; }
if (get_num(json, "mlp_bias", &d)) s->mlp_bias = d != 0;

/* embedding scale: gemma scales embeddings by sqrt(hidden_size) */
Expand Down Expand Up @@ -601,6 +601,17 @@ void q4nx_semantics_layer_mix(const Q4nxSemantics* s, char* out, size_t out_sz)
snprintf(out, out_sz, "full=%d sliding=%d linear=%d other=%d", full, slid, lin, oth);
}

int q4nx_semantics_attention_bias_from_tensors(Q4nxSemantics* s, const unsigned char* has_bias, int layers) {
if (!s || !has_bias || layers <= 0 || s->attention_bias) return -1;
for (int l = 0; l < layers; l++)
if (has_bias[l]) {
if (s->attention_bias_stated) return -2;
s->attention_bias = 1;
return l;
}
return -1;
}

int q4nx_semantics_qk_norm_from_tensors(Q4nxSemantics* s, const unsigned char* has_q, const unsigned char* has_k, int layers) {
if (!s || !has_q || !has_k || layers <= 0 || s->qk_norm) return -1;
for (int l = 0; l < layers; l++)
Expand Down
80 changes: 74 additions & 6 deletions tests/batch_check_state_ci.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
# limitations under the License.
"""The batch-check state pin (tests/batch_check_state_test.cpp, engine#1905/#1946) on a container made here.

usage: tests/batch_check_state_ci.py <batch_check_state_test binary> [lfm2|lfm3|qwen3_5] (needs numpy)
usage: tests/batch_check_state_ci.py <batch_check_state_test binary> [lfm2|lfm3|qwen2|qwen3_5] (needs numpy)

batch_check_state_test needs a Q4NX container, and CI has none, so on its own it skips (77) there and a regression of
#1909 -- the NPU_BATCH_CHECK oracle resetting the prefix it replays from -- lands green. This writes a two-layer qwen3
Expand Down Expand Up @@ -44,6 +44,12 @@
measured at engine#2254, corr 1.0000000 and max |d| 1.2e-05 of 7.63. A dead head gate (in both paths) gives corr 0.90
and another argmax, the usual partial RoPE 0.98, no sandwich 0.38; every one of them passes batched == sequential.

With `qwen2` it is a two-layer Qwen2 (engine#1786): q/k/v projection biases, no q/k norms, and a config.json with no
`attention_bias` key, which is how every Qwen2 config is written. With the key absent the forward loaded the biases
and never added them, so FastFlowLM's Qwen2.5-3B-Instruct-NPU2 answered noise. batched == sequential cannot see that
(both paths skip the biases alike), so the arm also compares the sequential logits with a numpy Qwen2 forward on the
container's own weights (qwen2_reference). Measured at the fix: see QWEN2_* below.

With `qwen3_5` it is a qwen3_5 hybrid (engine#1889): a linear-attention (GDN) layer, then a full-attention layer with
q/k norms, written under `language_model.*` so the packer takes its qwen3_5 path (renames, the conv1d transpose,
F32 `ssm_a` / `ssm_dt.bias`). The same rule, scoped to "not a conv layer", still refused every qwen3_5 hybrid: a GDN
Expand Down Expand Up @@ -91,6 +97,7 @@ def make_model(d, np, family="qwen3"):
rng = np.random.default_rng(1946)
lfm2 = family == "lfm2"
lfm3 = family == "lfm3"
qwen2 = family == "qwen2"
q35 = family == "qwen3_5"
layers = 3 if lfm2 else (4 if lfm3 else LAYERS)
conv_layers = (0, 2) if (lfm2 or lfm3) else ()
Expand Down Expand Up @@ -165,6 +172,12 @@ def norm(n):
p + "self_attn.q_norm.weight": norm(HEAD_DIM), p + "self_attn.k_norm.weight": norm(HEAD_DIM),
p + "mlp.gate_proj.weight": w(FFN, H), p + "mlp.up_proj.weight": w(FFN, H),
p + "mlp.down_proj.weight": w(H, FFN)})
if qwen2:
# Qwen2: biases on q/k/v (large enough to matter), no per-head q/k norms
del weights[p + "self_attn.q_norm.weight"], weights[p + "self_attn.k_norm.weight"]
weights.update({p + "self_attn.q_proj.bias": w(HEADS * HEAD_DIM, sc=0.5),
p + "self_attn.k_proj.bias": w(KV_HEADS * HEAD_DIM, sc=0.5),
p + "self_attn.v_proj.bias": w(KV_HEADS * HEAD_DIM, sc=0.5)})
os.makedirs(d)
write_safetensors(os.path.join(d, "model.safetensors"), weights)
cfg = {"architectures": ["Qwen3ForCausalLM"], "model_type": "qwen3", "hidden_size": H,
Expand All @@ -181,6 +194,8 @@ def norm(n):
del cfg["rms_norm_eps"]
cfg.update({"architectures": ["Lfm2ForCausalLM"], "model_type": "lfm2", "num_hidden_layers": layers,
"norm_eps": 1e-5, "conv_L_cache": 3, "full_attn_idxs": [1]})
if qwen2:
cfg.update({"architectures": ["Qwen2ForCausalLM"], "model_type": "qwen2"}) # and no attention_bias key
if lfm3:
del cfg["rms_norm_eps"], cfg["rope_theta"]
cfg.update({"architectures": ["Lfm3ForCausalLM"], "model_type": "lfm3", "num_hidden_layers": layers,
Expand Down Expand Up @@ -276,6 +291,59 @@ def rope(x, layer_type):
return rms(x, W("model.embedding_norm.weight"))[-1] @ W("lm_head.weight").T


def qwen2_reference(q4nx_dir, np, ids, shapes):
"""HF's Qwen2 forward (llama with q/k/v biases) on the container's own weights; the last position's logits."""
sys.path.insert(0, os.path.join(HERE, "..", "tools"))
from q4nx_pack import untile
with open(os.path.join(q4nx_dir, "config.json")) as f:
cfg = json.load(f)
with open(os.path.join(q4nx_dir, "model.q4nx"), "rb") as f:
n = struct.unpack("<Q", f.read(8))[0]
table = json.loads(f.read(n))
blob = f.read()

def W(name):
e = table[name]
lo, hi = e["data_offsets"]
if e["dtype"] == "BF16":
return (np.frombuffer(blob[lo:hi], dtype=np.uint16).astype(np.uint32) << 16).view(np.float32).reshape(e["shape"])
if e["dtype"] == "F32":
return np.frombuffer(blob[lo:hi], dtype=np.float32).reshape(e["shape"])
rows, k = shapes[name]
return untile(np.frombuffer(blob[lo:hi], dtype=np.uint8).reshape(-1, 5120), rows, k)

nh, nkv, hd, eps, T = cfg["num_attention_heads"], cfg["num_key_value_heads"], cfg["head_dim"], cfg["rms_norm_eps"], len(ids)

def rms(x, w):
return w * (x / np.sqrt((x.astype(np.float64) ** 2).mean(-1, keepdims=True) + eps)).astype(np.float32)

def rope(x):
inv = 1.0 / cfg["rope_theta"] ** (np.arange(0, hd, 2) / hd)
fr = np.arange(T)[:, None] * inv[None, :]
c, sn = np.cos(np.concatenate([fr, fr], -1))[:, None], np.sin(np.concatenate([fr, fr], -1))[:, None]
x = x.astype(np.float64)
return (x * c + np.concatenate([-x[..., hd // 2:], x[..., :hd // 2]], -1) * sn).astype(np.float32)

x = W("model.embed_tokens.weight")[ids].astype(np.float32)
for layer in range(cfg["num_hidden_layers"]):
p = f"model.layers.{layer}."
xn = rms(x, W(p + "input_layernorm.weight"))
q = (xn @ W(p + "self_attn.q_proj.weight").T + W(p + "self_attn.q_proj.bias")).reshape(T, nh, hd)
k = (xn @ W(p + "self_attn.k_proj.weight").T + W(p + "self_attn.k_proj.bias")).reshape(T, nkv, hd)
v = (xn @ W(p + "self_attn.v_proj.weight").T + W(p + "self_attn.v_proj.bias")).reshape(T, nkv, hd)
q, k = rope(q), rope(k)
k, v = np.repeat(k, nh // nkv, 1), np.repeat(v, nh // nkv, 1)
sc = np.einsum("thd,shd->hts", q, k) / np.sqrt(hd)
sc = np.where(np.tril(np.ones((T, T), bool))[None], sc, -np.inf)
sc = np.exp(sc - sc.max(-1, keepdims=True))
ctx = np.einsum("hts,shd->thd", sc / sc.sum(-1, keepdims=True), v)
x = x + ctx.reshape(T, nh * hd) @ W(p + "self_attn.o_proj.weight").T
h = rms(x, W(p + "post_attention_layernorm.weight"))
a = h @ W(p + "mlp.gate_proj.weight").T
x = x + ((a / (1.0 + np.exp(-a))) * (h @ W(p + "mlp.up_proj.weight").T)) @ W(p + "mlp.down_proj.weight").T
return rms(x, W("model.norm.weight"))[-1] @ W("lm_head.weight").T


def tmp_root():
"""tests/tmp_root.h's rule (engine#2029): scratch goes on disk, not in tmpfs. ONEBIT_TEST_TMP if set, else
~/.cache/scratch-tests/tmp, else the system temp directory."""
Expand All @@ -294,7 +362,7 @@ def tmp_root():


def main():
if len(sys.argv) not in (2, 3) or (len(sys.argv) == 3 and sys.argv[2] not in ("lfm2", "lfm3", "qwen3_5")):
if len(sys.argv) not in (2, 3) or (len(sys.argv) == 3 and sys.argv[2] not in ("lfm2", "lfm3", "qwen2", "qwen3_5")):
print(__doc__.strip().splitlines()[2])
return 2
family = sys.argv[2] if len(sys.argv) == 3 else "qwen3"
Expand All @@ -312,7 +380,7 @@ def main():
return 1
env = dict(os.environ, ONEBIT_TEST_Q4NX=out)
logits_out = os.path.join(tmp, "sequential.f32")
if family == "lfm3":
if family in ("lfm3", "qwen2"):
env["ONEBIT_TEST_LOGITS_OUT"] = logits_out
env.pop("NPU_INFER_HOST_GEMM", None) # init_host is host-GEMM already; nothing else should steer it
r = subprocess.run([sys.argv[1]], env=env, capture_output=True, text=True, timeout=110)
Expand All @@ -325,13 +393,13 @@ def main():
# arm, not a reason to skip it.
print("FAIL batch_check_state_test skipped on the fabricated container (init_host refused it)")
return 1
if family == "lfm3" and r.returncode == 0:
if family in ("lfm3", "qwen2") and r.returncode == 0:
ours = np.fromfile(logits_out, dtype=np.float32)[:VOCAB]
ref = lfm3_reference(out, np, [1, 2, 3, 4, 5, 6], shapes)
ref = (lfm3_reference if family == "lfm3" else qwen2_reference)(out, np, [1, 2, 3, 4, 5, 6], shapes)
corr = float(np.corrcoef(ours, ref)[0, 1])
ok = (corr >= LFM3_MIN_CORR and float(np.abs(ours - ref).max()) <= LFM3_MAX_DIFF * float(np.abs(ref).max())
and int(np.argmax(ours)) == int(np.argmax(ref)))
print(f"{'PASS' if ok else 'FAIL'} sequential logits vs the modeling_lfm3 reference on the container's "
print(f"{'PASS' if ok else 'FAIL'} sequential logits vs the {family} reference on the container's "
f"weights: corr {corr:.7f} (needs {LFM3_MIN_CORR}), max |d| {float(np.abs(ours - ref).max()):.3g} of "
f"{float(np.abs(ref).max()):.3g}, argmax {int(np.argmax(ours))} vs "
f"{int(np.argmax(ref))}")
Expand Down
21 changes: 21 additions & 0 deletions tests/q4nx_qk_norm_test.c
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,27 @@ int main(void) {
"(LFM2's conv layers, qwen3_5's linear-attention layers)");
check(q4nx_qk_gain_missing(0, 1, 0) == 0, "q/k norm off: no gain is expected anywhere");

/* engine#1786: attention_bias from the container, as qk_norm. Qwen2's config.json never states the key and every
* layer carries q/k/v biases; with the key absent they were loaded and never added (Qwen2.5-3B answered noise). */
{
const unsigned char none[3] = {0, 0, 0}, l1[3] = {0, 1, 1};
Q4nxSemantics b;
memset(&b, 0, sizeof b);
check(q4nx_semantics_attention_bias_from_tensors(&b, l1, 3) == 1 && b.attention_bias == 1,
"attention_bias absent from config.json, layer 1 carries biases: on, and layer 1 is named");
memset(&b, 0, sizeof b);
check(q4nx_semantics_attention_bias_from_tensors(&b, none, 3) == -1 && b.attention_bias == 0,
"absent, and no layer carries a bias: stays off");
memset(&b, 0, sizeof b);
b.attention_bias_stated = 1;
check(q4nx_semantics_attention_bias_from_tensors(&b, l1, 3) == -2 && b.attention_bias == 0,
"config.json says attention_bias=false and the container carries biases: a disagreement (-2, refused)");
memset(&b, 0, sizeof b);
b.attention_bias = b.attention_bias_stated = 1;
check(q4nx_semantics_attention_bias_from_tensors(&b, l1, 3) == -1 && b.attention_bias == 1,
"stated true: left as stated");
}

char path[512];
snprintf(path, sizeof path, "%s/config.json", dir);
unlink(path);
Expand Down
Loading