From dd9719b6601761dba7e69412ece77f3bc0064b6f Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Mon, 9 Nov 2026 05:56:42 -0400 Subject: [PATCH 01/14] npu-pack: a config.json that declares an optional field as null means unset, not unreadable (Refs #1503) `1bit npu-pack info` on `amd/Mistral-7B-Instruct-v0.2_rai_1.9.0_npu` (8,076,781,882 B) exits 1 with `[json.exception.type_error.302] type must be number, but is null` and reports no geometry at all. The pack is fine: its `config.json` declares `"head_dim": null` (and `"sliding_window": null`), and its `genai_config.json` states head_size 128, hidden 4096, heads 32, kv_heads 8, layers 32, vocab 32000 - config.json agrees on everything except that null. The cause is one nlohmann semantic: `json::value(key, fallback)` returns the fallback when the key is ABSENT but throws when the key is present and null. `read_dims` used it for every optional field, so `"head_dim": null` - which means "unset, use the default" in a pack written by AMD's exporter - became a hard failure before the reader ever looked at a tensor, and the default `hidden / heads` (4096/32 = 128, the value genai_config.json also states) was never reached. Reproduced identically on two builds, so this is not staleness: Oct-31 `build-oga` and Nov-5 `84fc1b3`. `opt_field()` treats null exactly like absent, and every optional read now goes through it: `num_key_value_heads`, `head_dim`, `vocab_size`, `rope_theta`, `model_type`, `rms_norm_eps` in `read_dims`, and `tie_word_embeddings` in `import_pack`. A wrong TYPE still throws inside `get()` - `"head_dim": "128"` is a real disagreement, not a default, and stays refused. `tests/npu_pack_import_test.cpp` gains `test_null_optional_dims` (ctest `npu_pack_import`): every optional field set to null falls back to its documented default (head_dim to hidden/heads, kv_heads to heads, rope_theta 10000, vocab 0, eps 1e-6, model_type empty), an explicit `head_dim` still wins, and a string `head_dim` is still refused. `read_dims` is exported as `read_dims_exported` the way `bf16_bits_of` already is, so the test needs no pack and no vendor stack. Why it matters beyond this pack: the geometry it unblocks is h4096 / 32L / NH32 / NKV8 / HD128 / IM14336 - the largest family in #1503's missing-artifact list, 31 packs (Llama-3-8B, Llama-3.1-8B and Instruct, Mistral-7B v0.2/v0.3, Hunyuan-7B), which this null currently stops at `info`. --- npu/pack_import.cpp | 30 ++++++++++++++++++++++------- npu/pack_import.h | 8 ++++++++ tests/npu_pack_import_test.cpp | 35 ++++++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 7 deletions(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index eb8118c4d..9d190029a 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -761,18 +761,30 @@ int dims_required(const json& c, const char* key, const std::string& source) { return c[key].get(); } +// An optional field, read the way a pack means it. nlohmann's value() throws when the key is +// present AND null (`type must be number, but is null`), which is not absence: AMD's 1.9 +// Mistral-7B-Instruct-v0.2 pack writes `"head_dim": null` (and `"sliding_window": null`) to mean +// "unset, use the default", and the throw turned an 8 GB pack into "info failed" with no geometry +// at all (engine#1650). A wrong TYPE still throws inside get(): that is a real disagreement, +// not a default. +template +T opt_field(const json& c, const char* key, T fallback) { + const auto it = c.find(key); + return (it == c.end() || it->is_null()) ? fallback : it->get(); +} + Dims read_dims(const json& c, const std::string& source = "config.json") { Dims d; d.hidden = dims_required(c, "hidden_size", source); d.layers = dims_required(c, "num_hidden_layers", source); d.heads = dims_required(c, "num_attention_heads", source); - d.kv_heads = c.value("num_key_value_heads", d.heads); - d.head_dim = c.value("head_dim", d.hidden / d.heads); + d.kv_heads = opt_field(c, "num_key_value_heads", d.heads); + d.head_dim = opt_field(c, "head_dim", d.hidden / d.heads); d.intermediate = dims_required(c, "intermediate_size", source); - d.vocab = c.value("vocab_size", 0); - d.rope_theta = c.value("rope_theta", 10000.0f); - d.model_type = c.value("model_type", ""); - d.rms_norm_eps = c.value("rms_norm_eps", 1e-6f); + d.vocab = opt_field(c, "vocab_size", 0); + d.rope_theta = opt_field(c, "rope_theta", 10000.0f); + d.model_type = opt_field(c, "model_type", ""); + d.rms_norm_eps = opt_field(c, "rms_norm_eps", 1e-6f); d.eps_bits = bf16_bits_of(d.rms_norm_eps); if (c.contains("eos_token_id")) { const auto& e = c["eos_token_id"]; @@ -784,6 +796,10 @@ Dims read_dims(const json& c, const std::string& source = "config.json") { } // namespace +// engine#1650: exported so the config reader can be tested on a pack that declares an optional +// field as null, without a pack and without the vendor stack. +Dims read_dims_exported(const json& c, const std::string& source) { return read_dims(c, source); } + // engine#734: exported so the epsilon anchor itself can be tested against a crafted .fconst, without a pack. // The two bytes a bf16 epsilon is stored as (little-endian): round the f32 to nearest-even bf16. unsigned short bf16_bits_of(float v) { @@ -1858,7 +1874,7 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons if (cf) { const auto c = json::parse(cf); tie_declared = c.contains("tie_word_embeddings"); - tied = c.value("tie_word_embeddings", false); + tied = opt_field(c, "tie_word_embeddings", false); } } bool head_written = false; diff --git a/npu/pack_import.h b/npu/pack_import.h index 87306bdf4..45700fb9e 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -35,6 +35,8 @@ #include #include + +#include #include #include @@ -292,6 +294,12 @@ struct FconstGeom { std::string why; // non-empty when the geometry could not be established }; unsigned short bf16_bits_of(float v); + +// engine#1650: config.json may declare an OPTIONAL field as JSON null (AMD's 1.9 +// Mistral-7B-Instruct-v0.2 pack writes `"head_dim": null`), which must read as "absent" and take +// the default rather than throw. Exported so tests/npu_pack_import_test.cpp can pin it without a +// pack; `source` names where the geometry came from, for the refusal messages. +Dims read_dims_exported(const nlohmann::json& c, const std::string& source); FconstGeom locate_fconst(const std::string& path, unsigned short eps_bits = 0x3586); int64_t attn_tile_bytes(int64_t k); diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index c66a66d34..9af0d5ef9 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -1721,6 +1721,40 @@ void test_weight_layout() { std::printf("weight layout: b8 allowed, pb16i/t2s/unknown refused\n"); } +// engine#1650: a config.json that declares an OPTIONAL field as JSON null means "unset", not +// "broken". AMD's 1.9 Mistral-7B-Instruct-v0.2 pack ships `"head_dim": null` and +// `"sliding_window": null`, and nlohmann's value() throws on a present-but-null key, so the reader +// reported `type must be number, but is null` and no geometry at all: an 8 GB pack named "info +// failed" with nothing said about what it contains. head_dim's default is the one every +// Llama-family config implies (hidden / heads); a WRONG TYPE must still be refused. +void test_null_optional_dims() { + using nlohmann::json; + const json base = {{"hidden_size", 4096}, {"num_hidden_layers", 32}, + {"num_attention_heads", 32}, {"intermediate_size", 14336}}; + json c = base; + for (const char* k : {"head_dim", "sliding_window", "num_key_value_heads", "rope_theta", + "vocab_size", "rms_norm_eps", "model_type"}) + c[k] = nullptr; // the measured pack, field for field + const pi::Dims d = pi::read_dims_exported(c, "config.json"); + check(d.head_dim == 128, "null head_dim defaults to hidden/heads (4096/32 = 128)"); + check(d.kv_heads == d.heads, "null num_key_value_heads defaults to the head count"); + check(d.rope_theta == 10000.0f, "null rope_theta defaults to 10000"); + check(d.vocab == 0, "null vocab_size reads as 0 (unset)"); + check(std::abs(d.rms_norm_eps - 1e-6f) < 1e-12f, "null rms_norm_eps defaults to 1e-6"); + check(d.model_type.empty(), "null model_type reads as empty, not as a crash"); + + json good = base; + good["head_dim"] = 128; // an explicit value still wins over the default + check(pi::read_dims_exported(good, "config.json").head_dim == 128, "an explicit head_dim wins"); + + json bad = base; + bad["head_dim"] = "128"; // null is absence; a string is a disagreement + bool threw = false; + try { (void)pi::read_dims_exported(bad, "config.json"); } catch (const std::exception&) { threw = true; } + check(threw, "a non-numeric head_dim is still refused"); + std::printf("null optional dims: head_dim falls back to hidden/heads, a wrong type still throws\n"); +} + void test_fconst_eps(const std::string& root) { const fs::path dir = fs::path(root) / "fconst-eps"; fs::remove_all(dir); fs::create_directories(dir); @@ -1798,6 +1832,7 @@ int main(int argc, char** argv) { test_text_artifacts(golden_dir.empty() ? "." : golden_dir); test_kernel_set_refusal(); test_weight_layout(); + test_null_optional_dims(); if (pack_ran) test_pack(pack, golden_dir); else std::printf("skip pack: no pack directory given (the crafted and golden legs only)\n"); } catch (const std::exception& e) { From 50cea8bc9bab17bb35cb946fdf57b5d746ff5d90 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Mon, 9 Nov 2026 07:20:35 -0400 Subject: [PATCH 02/14] npu-pack: the h4096 / 32-head / 32-layer Token .fconst block shape, measured against the 1.8 twin (Refs #1503) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With the null tolerance in the previous commit, `1bit npu-pack info` on `amd/Mistral-7B-Instruct-v0.2_rai_1.9.0_npu` reaches the next gate and stops there: …fconst: this is a Ryzen AI 1.9 Token .fconst (it carries 32 headers of 128 B, one per layer, 00 00 7f ff per head), and its block shape has not been measured for hidden 4096, 32 heads, 32 layers. … the 1.8 pack of the same model is what establishes it. It does. `tools/fconst19_measure.py` - the tool the table's own comment prescribes - against `amd/Mistral-7B-Instruct-v0.2_rai_1.8.0_npu_4K` gives, and verifies end to end: hidden 4096, heads 32, layers 32, prefix 12608, n_tiles 5, tile 2297856, rot 1, header_off 13795392, stride_1_9 117862720, 1954 of 3843419200 bytes differ (5.08e-05 %) <- content, not layout so the row is {4096, 32, 32, 12608, 5, 1, 2297856}. Two things in it were not derivable and would have been guessed wrong: the bundle is 5 tiles of 2,297,856 (11,489,280 B), which is NOT `header_off - prefix` - the header sits after the prefix and the bundle follows it - and `rot` is 1. Measured effect, same pack, same binary, in order: before rc 1, `json.exception.type_error.302: type must be number, but is null` after the null tolerance rc 1, the Token .fconst refusal above after this row rc 0, `gaps 0`, dims head_dim 128 / hidden 4096 / IM 14336 / NH 32 / 32 layers / NKV 8 / rope_theta 1e6 / vocab 32000 / model_type mistral and `npu-pack import -o … --weight-layout b8` then succeeds: **291 tensors, 867,968 q4_1 tiles**, container read back through `q4nx::dequant_tile` at **rel_err 0.000e+00, max |diff| 0**, attention **5120/5120** `.fconst` tiles and MLP **9216/9216** groups surviving decode->encode byte for byte, fp32 scale/zero exact for **100.00 %** of groups, and the importer's own stride read (117,862,592) matching the measured one. What this unblocks is the largest family on #1503: 31 packs at h4096 / NH32 / NKV8 / HD128 with IM 14336 (Llama-3-8B, Llama-3.1-8B and its Instruct twin, Mistral-7B v0.2/v0.3, Hunyuan-7B). Their remaining gate is the `_b8` kernel family `K4096N6144 / K4096N4096 / K4096N28672 / K14336N4096`, which is private-repo work and is not in this PR. --- npu/pack_import.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 9d190029a..b9e9ad62a 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -299,6 +299,13 @@ constexpr Fconst19Rule kFconst19Rules[] = { // prefix as Qwen3-1.7B at this hidden size (8,512), a different bundle (5 tiles of 591,872, not 3 of 1,183,744), // and 1,857 of 686,518,336 bytes differ (2.7e-6). This row is the one that disproved the QK-norm explanation above. {2048, 32, 16, 8512, 5, 1, 591872}, + // Mistral-7B-Instruct-v0.2 at hidden 4,096 / 32 heads / 32 layers: measured with tools/fconst19_measure.py against + // amd/Mistral-7B-Instruct-v0.2_rai_1.8.0_npu_4K, 1,954 of 3,843,419,200 bytes differ (5.08e-05) -- content, not + // layout. The prefix follows the Qwen3 line (64 + 2*hidden + 4,352 = 12,608), the bundle is 5 tiles of 2,297,856 + // (11,489,280 B, which is NOT header_off minus the prefix: the header sits after the prefix and the bundle follows + // it, so this row is one the arithmetic would have got wrong), and the header stride 117,862,720 is uniform over + // all 32 blocks. This is the largest family on #1503 (31 packs). + {4096, 32, 32, 12608, 5, 1, 2297856}, }; // The header a 1.9 block carries: 00 00 7f ff once per head. Its length being the head count is what makes From 6a8d00661f6b2023d96874d5b3892f0d75633c24 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 01:11:01 -0400 Subject: [PATCH 03/14] npu: the integer host arm a kernel gate needs, because f32 cannot be bit-exact with int8 (Refs #1808) The gate every kernel family owes is "bit-exact against the host reference on the device, 3 seeds plus an edge case, 3 repeats". The only host arm the forward offers is `NPU_INFER_HOST_GEMM=1`, which runs **f32** (`q4nx_host_gemm_f32_padded`); the device runs **int8 x int4**. Those two cannot be bit-exact by construction, so every gate written against them yields a quality number - cosine and argmax agreement - and never the bit-exactness the plan asks for. Measured tonight on a 4096 family: argmax identical for three steps at cosine 0.9955-0.9974, one flip on the fourth. `q4nx_host_gemm_i8()` is the missing arm: exact int32 accumulation of the same operands the kernel gets (int8 activation row, int4 codes as the importer packs them), with the scaling deliberately left to the caller so a scale mistake cannot hide inside the integer comparison. It sits beside the f32 function in `npu/forward/src/q4nx_pack.c` because the two answer the same contract. `tests/npu_host_gemm_i8_test.cpp` (ctest `npu_host_gemm_i8`) pins it against an **independently generated** known answer - `tools/host_gemm_i8_fixture.py` computes the expected int32 in Python from the operands it writes out, so a change that merely agrees with itself cannot pass - plus two properties worth naming: - every weight code in the fixture is an int4 value in [-8, 7], so a caller that hands the function a q4_1 byte instead of a code fails here rather than producing plausible garbage; - an activation of **-128** (outside the quantiser's [-127, 127], inside int8) accumulates exactly: the product bound for the largest K in use (14336) is 1.47e7, well inside int32, so an int8 intermediate or any int32 saturation would be a real bug and is now a red test. Host only, no device. Wiring this arm into the forward as `NPU_INFER_HOST_GEMM=i8` is the next commit; the default f32 path is untouched by this one. --- CMakeLists.txt | 8 ++ npu/forward/include/q4nx_pack.h | 7 ++ npu/forward/src/q4nx_pack.c | 19 ++++ tests/golden/host_gemm_i8_fixture.h | 157 ++++++++++++++++++++++++++++ tests/npu_host_gemm_i8_test.cpp | 82 +++++++++++++++ tools/host_gemm_i8_fixture.py | 70 +++++++++++++ 6 files changed, 343 insertions(+) create mode 100644 tests/golden/host_gemm_i8_fixture.h create mode 100644 tests/npu_host_gemm_i8_test.cpp create mode 100755 tools/host_gemm_i8_fixture.py diff --git a/CMakeLists.txt b/CMakeLists.txt index d4b72ecc4..d70b585da 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -792,6 +792,14 @@ if(Python3_Interpreter_FOUND) # the whole run loop against canned completions, and re-derives the summary from the written transcript. Stdlib # only, with no route, no socket, no dataset fetch and no device. add_test(NAME gsm8k_eval COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/gsm8k_eval_test.py) + # engine#1808: the kernel gate asks for "bit-exact against the host reference", but the only host arm the + # forward offers is f32 -- which cannot be bit-exact with an int8 x int4 device by construction. This pins + # the integer arm (q4nx_host_gemm_i8) against an independently generated known answer, including an + # activation of -128 so a naive int8 intermediate or an int32 saturation would show. Pure host, no device. + add_executable(npu_host_gemm_i8_test tests/npu_host_gemm_i8_test.cpp) + target_include_directories(npu_host_gemm_i8_test PRIVATE npu/forward/include ${CMAKE_SOURCE_DIR}/tests) + target_link_libraries(npu_host_gemm_i8_test PRIVATE onebit_npu) + add_test(NAME npu_host_gemm_i8 COMMAND npu_host_gemm_i8_test) # The repack's tile-order claim (engine#1110): `npu_plan.py` writes container.tile_order=engine-1 for every # plan, so the load-time guard (#1130) is green on a container whose TILES are another order. This pins the # mapping from q4nx_diff's verdict to (stamp value, fatal), stdlib only: no model, no numpy, no device. diff --git a/npu/forward/include/q4nx_pack.h b/npu/forward/include/q4nx_pack.h index 6621d90e5..f256652cf 100644 --- a/npu/forward/include/q4nx_pack.h +++ b/npu/forward/include/q4nx_pack.h @@ -170,6 +170,13 @@ void q4nx_pad_a_f32(const float* A, int M, int real_k, int K, float* out); void q4nx_host_gemm_f32_padded(const float* A, int a_k, int M, int K, int N, const float* Bmat, float* out); +/* The integer arm of the same host path (engine#1808): int8 activation x int4 weight codes -> exact int32, + * which is what the kernel computes. Written for the bit-exactness gate, because the f32 host path cannot + * be bit-exact with an int8 device arm by construction; scales are applied by the caller on both sides so + * a scale mistake cannot hide inside the integer comparison. `codes` is [K][N], one int8 per element + * holding an int4 value in [-8, 7], exactly as the importer's packB lays it down. */ +void q4nx_host_gemm_i8(const int8_t* a8, int K, const int8_t* codes, int N, int32_t* out32); + /* Architecture dims a design set is derived from. */ typedef struct { diff --git a/npu/forward/src/q4nx_pack.c b/npu/forward/src/q4nx_pack.c index 8354b5dfd..c10aeda5e 100644 --- a/npu/forward/src/q4nx_pack.c +++ b/npu/forward/src/q4nx_pack.c @@ -131,6 +131,25 @@ void q4nx_pad_a_f32(const float* A, int M, int real_k, int K, float* out) { } } +/* The integer arm of the same contract (engine#1808): the activation is already quantised to int8 by the + * engine's own rule (q4nx_quant_i8::row, which is byte-identical to its scalar reference), the weights are + * the container's int4 codes as the importer packed them, and the accumulation is EXACT int32 -- no float + * anywhere. That is what the kernel computes, so this is the only host reference a bit-exactness gate can + * be written against: the f32 path (below) cannot be bit-exact with int8 by construction. + * + * out32[n] = sum_k codes[k][n] * a8[k], codes in [-8, 7], a8 in [-127, 127]. + * Scaling (per-128-K group multiplier and the per-row activation scale) is deliberately NOT applied here: + * the gate compares the int32 exactly, and applies scales on both sides from the same inputs, so a scale + * mistake cannot hide inside the comparison. */ +void q4nx_host_gemm_i8(const int8_t* a8, int K, const int8_t* codes, int N, int32_t* out32) { + for (int n = 0; n < N; n++) out32[n] = 0; + for (int k = 0; k < K; k++) { + const int a = (int)a8[k]; + const int8_t* row = codes + (size_t)k * (size_t)N; + for (int n = 0; n < N; n++) out32[n] += (int32_t)((int)row[n] * a); + } +} + void q4nx_host_gemm_f32_padded(const float* A, int a_k, int M, int K, int N, const float* Bmat, float* out) { if (!A || !Bmat || !out || M <= 0 || K <= 0 || N <= 0) return; diff --git a/tests/golden/host_gemm_i8_fixture.h b/tests/golden/host_gemm_i8_fixture.h new file mode 100644 index 000000000..c9d968429 --- /dev/null +++ b/tests/golden/host_gemm_i8_fixture.h @@ -0,0 +1,157 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Generated by tools/host_gemm_i8_fixture.py -- do not edit. Known answer for +// tests/npu_host_gemm_i8_test.cpp: int8 x int4 -> int32, exact, no scales involved. +#pragma once +#include + +static constexpr int kHostI8K = 128; +static constexpr int kHostI8N = 4; + +static const int8_t kHostI8A[128] = {-51, 38, -83, 17, 87, 56, 58, -7, 59, 42, -32, -46, -81, 16, -44, 120, -69, 79, -44, 9, 12, -24, 9, -92, 115, 116, -37, 52, 24, -13, 59, -25, 81, 59, -70, -83, 17, 48, 0, 11, -75, -88, 116, -86, -75, -60, 62, -111, -109, -49, 112, -4, -114, -107, -80, 56, 65, 114, 113, -28, 40, -66, -6, 127, 103, -81, -105, -72, 12, 98, -8, -16, -106, 31, 71, -94, 78, 122, -120, -107, -58, -37, -90, 98, -72, 62, 79, 0, -50, -93, -109, -72, -115, 23, 101, -54, 24, -122, 85, 32, -47, -120, 97, -12, -77, 65, -64, 67, 35, -9, -29, 94, -84, 75, 50, -8, -78, -15, -124, 49, -57, -127, 77, -54, -45, -22, -80, 88}; + +static const int8_t kHostI8Codes[128][4] = { + {3, -7, -7, -8}, + {6, -7, -4, -6}, + {5, -6, -8, -1}, + {-4, 4, -7, 7}, + {-1, -3, -1, 2}, + {4, -6, 6, -6}, + {-7, -5, 4, -1}, + {-1, 2, -5, -6}, + {4, -2, 5, -5}, + {-2, -3, -6, 7}, + {-1, -3, 5, -7}, + {-3, 4, -8, -1}, + {-7, 3, -6, 3}, + {2, 7, 5, 5}, + {-4, 4, 0, -6}, + {-1, 7, -1, 0}, + {-7, -7, 1, -4}, + {1, 3, 1, -1}, + {-6, 4, 6, 7}, + {-5, 4, 1, 6}, + {-6, -8, -8, 7}, + {4, -1, 3, -7}, + {5, 4, -4, -8}, + {6, -1, -2, -2}, + {5, -3, -5, 0}, + {-4, 5, 5, -3}, + {-5, -1, -2, 4}, + {4, 4, 0, -5}, + {1, -5, -7, 7}, + {5, 1, -1, -8}, + {-2, -3, -7, -8}, + {-6, 3, 6, -1}, + {-8, 5, 6, 5}, + {2, 7, -2, 2}, + {7, -5, 5, -3}, + {5, -4, -3, -4}, + {3, -3, 2, -7}, + {-5, -2, 5, 3}, + {-6, -5, -3, -1}, + {2, 1, 3, 7}, + {-8, -7, -4, -5}, + {-4, 4, -4, 4}, + {1, 7, -3, -4}, + {0, 2, 5, 6}, + {0, -1, 4, 5}, + {1, 7, -8, -2}, + {-8, -6, 7, -3}, + {-4, -6, -3, 5}, + {-5, 7, -8, -4}, + {3, 0, 2, -7}, + {0, 2, 5, -3}, + {-5, -8, -7, 7}, + {3, -2, 2, 5}, + {-7, 7, -5, 4}, + {2, 3, 5, -4}, + {-3, -6, 3, 2}, + {2, 7, 7, 6}, + {-2, -4, -6, 1}, + {3, -5, -3, -5}, + {-6, -6, -6, -4}, + {2, -6, 5, 7}, + {3, -1, -3, 0}, + {-1, 6, -5, -7}, + {5, -8, 3, 3}, + {-4, -4, 0, -2}, + {3, -6, -1, 4}, + {5, 1, 7, -8}, + {-5, -2, 6, -4}, + {3, -2, 3, -7}, + {7, -5, -7, 2}, + {7, -6, 5, 2}, + {-6, -2, -3, -3}, + {-6, -8, 4, 6}, + {6, -6, 7, -7}, + {-6, 7, 2, 6}, + {-4, -4, -6, 7}, + {-2, -4, 6, 7}, + {4, 7, 3, -2}, + {1, 3, -3, 2}, + {-2, 6, -4, -3}, + {7, -5, -6, -6}, + {3, 3, -6, -8}, + {3, -7, -6, 2}, + {-6, -2, 2, 1}, + {-1, 2, 1, -2}, + {3, 1, 0, 1}, + {-3, -2, -4, -6}, + {1, 7, 4, 6}, + {-4, 1, 7, -1}, + {-7, -1, 7, 3}, + {-8, -8, 0, -7}, + {4, 5, 1, -5}, + {-3, 1, 7, -8}, + {6, -5, -8, 2}, + {-1, 6, 6, 0}, + {-5, 6, 0, 1}, + {4, -5, 1, -5}, + {4, -3, -3, -2}, + {7, 3, -3, -5}, + {5, -2, 0, -6}, + {-3, -4, 4, -7}, + {-4, -5, 6, -2}, + {-8, 3, -4, 2}, + {-7, -4, 7, -3}, + {1, -2, 2, 4}, + {2, -2, -2, -1}, + {-6, 6, -4, 1}, + {-8, 2, -5, -6}, + {0, 0, -2, -6}, + {5, 0, 2, -1}, + {4, 5, 1, -2}, + {-8, 6, 6, -1}, + {-5, 1, 4, -8}, + {3, 0, -3, 7}, + {5, -1, -5, -2}, + {-1, -7, -3, 4}, + {-5, -2, -5, -5}, + {6, 6, 6, 6}, + {-5, -6, 3, 5}, + {-4, 1, -3, 4}, + {0, 0, 0, 1}, + {-4, 4, 2, -7}, + {-1, -7, 1, 5}, + {-1, 3, -6, 6}, + {-5, 2, -4, -1}, + {5, 6, -2, 4}, + {-7, 5, 2, -2}, + {-8, -1, -7, -4}, +}; + +static const int32_t kHostI8Expected[4] = {4927, 1939, 1583, 2736}; diff --git a/tests/npu_host_gemm_i8_test.cpp b/tests/npu_host_gemm_i8_test.cpp new file mode 100644 index 000000000..f5093c59a --- /dev/null +++ b/tests/npu_host_gemm_i8_test.cpp @@ -0,0 +1,82 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// npu_host_gemm_i8_test: the known answer for the integer host arm (engine#1808). +// +// The gate for a kernel family asks for "bit-exact against the host reference". The only host arm the +// forward offers is f32, and f32 cannot be bit-exact with an int8 x int4 device by construction, so the +// comparison can only ever produce a quality number. `q4nx_host_gemm_i8()` is the missing arm: exact int32 +// accumulation of the same operands the kernel gets. +// +// This test is a *known answer*, not a re-implementation: the fixture in tests/golden/ was generated by an +// independent Python computation (tools/host_gemm_i8_fixture.py) with a fixed seed, so a change that +// "agrees with itself" cannot pass. Three arms: +// 1. the fixture: K=128, N=4, int8 activations and int4 codes, expected int32 per column; +// 2. the int4 range: every code in the fixture lies in [-8, 7] (a code outside it would mean the caller +// is handing us a q4_1 byte instead of a code, which is the mistake the whole gate exists to catch); +// 3. the edge: an activation of -128 (outside the quantiser's range, inside int8) must still accumulate +// exactly rather than saturate at the int32 level -- the product bound is 128 * 8 * K, which for the +// largest K in use (14336) is 1.47e7, far inside int32, so a saturation here would be a real bug. + +#include "q4nx_pack.h" + +#include +#include + +#include "golden/host_gemm_i8_fixture.h" + +static int failures = 0; + +static void check(bool ok, const char* what) { + std::printf("%s %s\n", ok ? "ok " : "FAIL", what); + if (!ok) failures++; +} + +int main() { + // 1. the recorded known answer + int32_t got[kHostI8N] = {0, 0, 0, 0}; + q4nx_host_gemm_i8(kHostI8A, kHostI8K, &kHostI8Codes[0][0], kHostI8N, got); + bool same = true; + for (int n = 0; n < kHostI8N; n++) { + if (got[n] != kHostI8Expected[n]) { + std::printf(" column %d: got %d, expected %d\n", n, (int)got[n], (int)kHostI8Expected[n]); + same = false; + } + } + check(same, "int8 x int4 -> int32 matches the independent Python fixture, all 4 columns"); + + // 2. the fixture only carries int4 codes + bool in_range = true; + for (int k = 0; k < kHostI8K && in_range; k++) + for (int n = 0; n < kHostI8N; n++) + if (kHostI8Codes[k][n] < -8 || kHostI8Codes[k][n] > 7) in_range = false; + check(in_range, "every weight code is an int4 value in [-8, 7]"); + + // 3. the int32 edge: a naive int8 multiply would overflow at 128 * 127, this must not + const int K = 256; + static int8_t a8[256]; + static int8_t codes[256 * 2]; + for (int k = 0; k < K; k++) { a8[k] = (k == 0) ? -128 : 127; codes[k * 2] = -8; codes[k * 2 + 1] = 7; } + int32_t edge[2] = {0, 0}; + q4nx_host_gemm_i8(a8, K, codes, 2, edge); + // column 0: -8 * (-128) + -8 * 127 * 255 = 1024 - 259080 = -258056 + // column 1: 7 * (-128) + 7 * 127 * 255 = -896 + 226695 = 225799 + check(edge[0] == -258056 && edge[1] == 225799, + "a -128 activation accumulates exactly (no int8 intermediate, no int32 saturation)"); + + if (failures) { std::fprintf(stderr, "%d check(s) failed\n", failures); return 1; } + std::printf("all npu_host_gemm_i8 checks passed\n"); + return 0; +} diff --git a/tools/host_gemm_i8_fixture.py b/tools/host_gemm_i8_fixture.py new file mode 100755 index 000000000..049e8d1e5 --- /dev/null +++ b/tools/host_gemm_i8_fixture.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Write tests/golden/host_gemm_i8_fixture.h — the known answer for q4nx_host_gemm_i8(). + +engine#1808: the kernel gate asks for "bit-exact against the host reference", but the only host arm the +forward offers is f32 (`q4nx_host_gemm_f32_padded`), and f32 cannot be bit-exact with an int8 x int4 +device by construction. The integer arm (`q4nx_host_gemm_i8`) accumulates exactly, and this fixture is its +independent known answer: the expected int32 values are computed here, in Python, from the operands that +are also written out, so a C++ change that merely agrees with itself cannot pass the test. + + tools/host_gemm_i8_fixture.py > tests/golden/host_gemm_i8_fixture.h + +Fixed seed, so the file is reproducible; K=128 (the smallest shape that reaches a whole 128-K group) and +N=4 columns, with activations across the quantiser's full [-127, 127] range and codes across [-8, 7]. +""" +import random +import sys + +SEED = 20261110 +K, N = 128, 4 + + +def main(): + rng = random.Random(SEED) + a8 = [rng.randint(-127, 127) for _ in range(K)] + codes = [[rng.randint(-8, 7) for _ in range(N)] for _ in range(K)] # int4 value held in an int8 + expect = [sum(codes[k][n] * a8[k] for k in range(K)) for n in range(N)] + w = sys.stdout.write + w("// Copyright 2026 bong-water-water-bong\n") + w("// SPDX-License-Identifier: Apache-2.0\n") + w("//\n") + w("// Licensed under the Apache License, Version 2.0 (the \"License\");\n") + w("// you may not use this file except in compliance with the License.\n") + w("// You may obtain a copy of the License at\n") + w("//\n") + w("// http://www.apache.org/licenses/LICENSE-2.0\n") + w("//\n") + w("// Unless required by applicable law or agreed to in writing, software\n") + w("// distributed under the License is distributed on an \"AS IS\" BASIS,\n") + w("// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n") + w("// See the License for the specific language governing permissions and\n") + w("// limitations under the License.\n") + w("//\n") + w("// Generated by tools/host_gemm_i8_fixture.py -- do not edit. Known answer for\n") + w("// tests/npu_host_gemm_i8_test.cpp: int8 x int4 -> int32, exact, no scales involved.\n") + w("#pragma once\n#include \n\n") + w(f"static constexpr int kHostI8K = {K};\nstatic constexpr int kHostI8N = {N};\n\n") + w(f"static const int8_t kHostI8A[{K}] = {{{', '.join(str(v) for v in a8)}}};\n\n") + w(f"static const int8_t kHostI8Codes[{K}][{N}] = {{\n") + for k in range(K): + w(" {%s},\n" % ", ".join(str(v) for v in codes[k])) + w("};\n\n") + w(f"static const int32_t kHostI8Expected[{N}] = {{{', '.join(str(v) for v in expect)}}};\n") + + +if __name__ == "__main__": + main() From be7e12876cf7421943bcd24737d659eacb74b5a5 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 01:22:26 -0400 Subject: [PATCH 04/14] npu: NPU_INFER_HOST_GEMM=i8 selects the integer arm, so a family can be gated bit-exactly (Refs #1808) The arm added in the previous commit was unreachable: the forward had one switch, and any value of NPU_INFER_HOST_GEMM meant the f32 path. Now "i8" selects the integer arm at both host-GEMM call sites (single row and rows/padded form), and every other value keeps f32 -- so no existing caller, row or measurement moves. `q4nx_host_gemm_i8_from_f32()` is the whole-GEMM form: it quantises the activation row with the engine's own rule, quantises each weight column per 128-K group with the same rule, clamps the codes to the int4 range [-8, 7], and accumulates EXACT int32 -- no float anywhere in the sum. The caller applies the two factors the kernel's epilogue applies. **What this arm can and cannot prove, stated in the header because it decides which gate it serves:** the weights are quantised here from the dequantised f32 matrix, not read from the container's packed b8 image, so this arm does NOT see a wrong weight layout (the f32 arm covers that by construction). What it does see is a wrong int4 quantisation, a wrong 128-K group boundary, an int8 intermediate or a saturated accumulator -- i.e. the kernel's arithmetic, which is the half that "cos 0.9955 against f32" cannot resolve. Test: a fourth arm computes the same GEMM independently in double from the same rule and requires exact int32 equality, at **K=130** so the 128-K group boundary is exercised (one full group plus a 2-row tail), and M=2 so the row stride is too. Built clean, test green, copyright --check rc 0. --- npu/forward/include/q4nx_forward.h | 5 ++++ npu/forward/include/q4nx_pack.h | 7 +++++ npu/forward/src/q4nx_forward.cpp | 23 +++++++++++++- npu/forward/src/q4nx_pack.c | 48 ++++++++++++++++++++++++++++++ tests/npu_host_gemm_i8_test.cpp | 44 +++++++++++++++++++++++++++ 5 files changed, 126 insertions(+), 1 deletion(-) diff --git a/npu/forward/include/q4nx_forward.h b/npu/forward/include/q4nx_forward.h index 16421ce53..a967bfd9c 100644 --- a/npu/forward/include/q4nx_forward.h +++ b/npu/forward/include/q4nx_forward.h @@ -235,6 +235,11 @@ class Q4nxNpuForward { bool init_impl(xrt::device* dev, const char* model_path, const std::string& artifact_dir); bool host_gemm_ = false; + // NPU_INFER_HOST_GEMM=i8 (engine#1808): the host arm accumulates int8 x int4 in EXACT int32, which is + // the only arm a kernel family can be bit-exact against -- f32 cannot match an int8 device by + // construction. It cannot see a wrong weight layout (the f32 arm covers that); it can see a wrong + // quantisation, a wrong 128-K group boundary, an int8 intermediate or a saturated accumulator. + bool host_gemm_i8_ = false; bool load_layer(int l); // `a_k` is the activation's REAL width per row (the design's real_k, i.e. the model's hidden or // intermediate size). It is K whenever the model is already on the container's 256-column K diff --git a/npu/forward/include/q4nx_pack.h b/npu/forward/include/q4nx_pack.h index f256652cf..2a296a43c 100644 --- a/npu/forward/include/q4nx_pack.h +++ b/npu/forward/include/q4nx_pack.h @@ -177,6 +177,13 @@ void q4nx_host_gemm_f32_padded(const float* A, int a_k, int M, int K, int N, con * holding an int4 value in [-8, 7], exactly as the importer's packB lays it down. */ void q4nx_host_gemm_i8(const int8_t* a8, int K, const int8_t* codes, int N, int32_t* out32); +/* The whole GEMM in int8 x int4 for the gate's `i8` host arm, quantising both sides here with the engine's + * own rule. It cannot see a wrong weight LAYOUT (the f32 arm covers that); what it can see is a wrong int4 + * quantisation, a wrong 128-K group boundary, an int8 intermediate or a saturated accumulator -- i.e. the + * kernel's arithmetic, exactly. int32 out, M*N, no scaling applied. */ +void q4nx_host_gemm_i8_from_f32(const float* A, int a_k, int M, int K, int N, const float* Bmat, + int32_t* out32); + /* Architecture dims a design set is derived from. */ typedef struct { diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index e60a4a95d..15549782f 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -410,7 +410,14 @@ bool Q4nxNpuForward::init_host(const char* model_path, const std::string& dir) { bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, const std::string& dir) { - if (getenv("NPU_INFER_HOST_GEMM")) host_gemm_ = true; + if (const char* hg = getenv("NPU_INFER_HOST_GEMM")) { + host_gemm_ = true; + // engine#1808: "i8" selects the integer arm, whose int32 accumulation is the only thing a + // kernel family can be bit-exact against (an int8 device cannot match an f32 host by + // construction). Any other value keeps the f32 path, which is what every existing caller and + // every row measured so far used. + host_gemm_i8_ = (hg[0] == 'i' && hg[1] == '8'); + } // The launch layer. A1: everything below that touches a device talks to this, and the settings the // container declares (weight layout, pb16i image dir, Hadamard fold) are pushed into it as they are // parsed. A2 turns this into NpuTarget::open(plan, kernels_dir). @@ -2966,6 +2973,14 @@ void Q4nxNpuForward::npu_gemm(int role, const float* A, int K, int N, const floa // unpadded A. (In review of this PR the split moved the pad into the target and left this // branch reading past the caller's buffer at a padded design.) out.assign((size_t)N, 0.0f); + if (host_gemm_i8_) { + // int32 arm: quantise both sides here, accumulate exactly, then apply the same two factors the + // kernel's epilogue applies (the activation row scale and the per-128-K weight multiplier). + std::vector acc((size_t)N, 0); + q4nx_host_gemm_i8_from_f32(A, a_k, 1, K, N, Bmat, acc.data()); + for (int n = 0; n < N; n++) out[(size_t)n] = (float)acc[(size_t)n]; + return; + } q4nx_host_gemm_f32_padded(A, a_k, 1, K, N, Bmat, out.data()); return; } @@ -2985,6 +3000,12 @@ void Q4nxNpuForward::npu_gemm_rows(int role, const float* A, int M, int K, int N // Same pad, and here it is also a stride: the rows form used to index A at `m * K` while the // caller's buffer is laid out at `m * a_k`, so every row past the first read the wrong row. out.assign((size_t)M * N, 0.0f); + if (host_gemm_i8_) { + std::vector acc((size_t)M * (size_t)N, 0); + q4nx_host_gemm_i8_from_f32(A, a_k, M, K, N, Bmat, acc.data()); + for (size_t i = 0; i < acc.size(); i++) out[i] = (float)acc[i]; + return; + } q4nx_host_gemm_f32_padded(A, a_k, M, K, N, Bmat, out.data()); return; } diff --git a/npu/forward/src/q4nx_pack.c b/npu/forward/src/q4nx_pack.c index c10aeda5e..d14f936fd 100644 --- a/npu/forward/src/q4nx_pack.c +++ b/npu/forward/src/q4nx_pack.c @@ -141,6 +141,54 @@ void q4nx_pad_a_f32(const float* A, int M, int real_k, int K, float* out) { * Scaling (per-128-K group multiplier and the per-row activation scale) is deliberately NOT applied here: * the gate compares the int32 exactly, and applies scales on both sides from the same inputs, so a scale * mistake cannot hide inside the comparison. */ +/* The whole GEMM in int8 x int4, for the gate's `NPU_INFER_HOST_GEMM=i8` arm: quantise the activation row + * and every weight column with the engine's own rule, accumulate exactly, and hand back the int32 -- the + * caller applies the multiplier and the activation scale, the same two factors the kernel's epilogue uses. + * + * LIMIT, stated because it decides what this arm can and cannot prove: the weights are quantised HERE from + * the dequantised f32 matrix, not read from the container's packed b8 image. So this arm isolates the + * kernel's ACCUMULATION and epilogue from its weight-layout handling: a wrong tile order will not show up + * here (the f32 arm covers that by construction), but a wrong int4 quantisation, a wrong per-128-K group + * boundary, an int8 intermediate or a saturated accumulator will. That is the split the MoE oracle already + * uses (`expected_int32` with the scales factored out), and it is what makes an int32 bit-exact comparison + * meaningful at all. */ +void q4nx_host_gemm_i8_from_f32(const float* A, int a_k, int M, int K, int N, const float* Bmat, + int32_t* out32) { + const int G = 128; /* the arena's multiplier group: one d per 128 K-rows per column */ + const int kg = (K + G - 1) / G; + for (int m = 0; m < M; m++) { + const float* a = A + (size_t)m * (size_t)a_k; + int32_t* out = out32 + (size_t)m * (size_t)N; + for (int n = 0; n < N; n++) out[n] = 0; + for (int g = 0; g < kg; g++) { + const int k0 = g * G, k1 = (k0 + G < K) ? k0 + G : K; + /* the activation's own per-row scale is the caller's; here only the integer codes matter, and the + * kernel receives them already scaled, so quantise with a per-group unit scale to keep the + * comparison about the accumulation rather than about a float that both sides share */ + for (int n = 0; n < N; n++) { + float amax = 0.0f; + for (int k = k0; k < k1; k++) { + const float v = fabsf(Bmat[(size_t)k * (size_t)N + (size_t)n]); + if (v > amax && isfinite(v)) amax = v; + } + if (!(amax >= 1e-12f)) amax = 1.0f; + const float tis = 127.0f / amax; + for (int k = k0; k < k1; k++) { + float v = Bmat[(size_t)k * (size_t)N + (size_t)n]; + if (!isfinite(v)) v = 0.0f; + int q = (int)roundf(v * tis); + if (q > 7) q = 7; + else if (q < -8) q = -8; /* int4 codes are [-8, 7] in this format */ + int aq = (int)roundf(a[k] * 127.0f); + if (aq > 127) aq = 127; + else if (aq < -127) aq = -127; + out[n] += (int32_t)(q * aq); + } + } + } + } +} + void q4nx_host_gemm_i8(const int8_t* a8, int K, const int8_t* codes, int N, int32_t* out32) { for (int n = 0; n < N; n++) out32[n] = 0; for (int k = 0; k < K; k++) { diff --git a/tests/npu_host_gemm_i8_test.cpp b/tests/npu_host_gemm_i8_test.cpp index f5093c59a..4339b4047 100644 --- a/tests/npu_host_gemm_i8_test.cpp +++ b/tests/npu_host_gemm_i8_test.cpp @@ -34,6 +34,9 @@ #include #include +#include +#include +#include #include "golden/host_gemm_i8_fixture.h" @@ -76,6 +79,47 @@ int main() { check(edge[0] == -258056 && edge[1] == 225799, "a -128 activation accumulates exactly (no int8 intermediate, no int32 saturation)"); + // 4. the whole-GEMM arm the gate switches on: quantise both sides here, accumulate exactly, and check + // the two properties that make the comparison meaningful -- the per-128-K group boundary is honoured + // (a K of 130 must not fold the last two rows into the first group's scale) and int4 codes stay in + // [-8, 7] so an out-of-range code cannot silently wrap. + { + const int KK = 130, NN = 3, MM = 2; + std::vector A((size_t)MM * KK), B((size_t)KK * NN); + for (int m = 0; m < MM; m++) + for (int k = 0; k < KK; k++) A[(size_t)m * KK + k] = (float)((k * 7 + m * 3) % 11) - 5.0f; + for (int k = 0; k < KK; k++) + for (int n = 0; n < NN; n++) B[(size_t)k * NN + n] = (float)((k * 5 + n * 13) % 17) / 3.0f; + std::vector out((size_t)MM * NN, 0); + q4nx_host_gemm_i8_from_f32(A.data(), KK, MM, KK, NN, B.data(), out.data()); + // independent computation in long double, from the same quantisation rule + bool ok = true; + for (int m = 0; m < MM && ok; m++) { + for (int n = 0; n < NN && ok; n++) { + double want = 0; + for (int g = 0; g < 2; g++) { // 128 + 2 rows = two groups + const int k0 = g * 128, k1 = (k0 + 128 < KK) ? k0 + 128 : KK; + double amax = 0; + for (int k = k0; k < k1; k++) amax = std::max(amax, std::fabs((double)B[(size_t)k * NN + n])); + if (amax < 1e-12) amax = 1; + const double tis = 127.0 / amax; + for (int k = k0; k < k1; k++) { + int q = (int)std::round((double)B[(size_t)k * NN + n] * tis); + q = q > 7 ? 7 : (q < -8 ? -8 : q); + int aq = (int)std::round((double)A[(size_t)m * KK + k] * 127.0); + aq = aq > 127 ? 127 : (aq < -127 ? -127 : aq); + want += (double)q * (double)aq; + } + } + if ((double)out[(size_t)m * NN + n] != want) { + std::printf(" m=%d n=%d: got %d, want %.0f\n", m, n, (int)out[(size_t)m * NN + n], want); + ok = false; + } + } + } + check(ok, "whole-GEMM i8 arm: exact int32, 128-K group boundary respected at K=130"); + } + if (failures) { std::fprintf(stderr, "%d check(s) failed\n", failures); return 1; } std::printf("all npu_host_gemm_i8 checks passed\n"); return 0; From 75007493a29991c294f36a1bf39a1643820b7e97 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 02:00:56 -0400 Subject: [PATCH 05/14] npu: cache the integer arm's weight codes per tensor, so a 7B token stops costing 7.5 G scalar ops (Refs #1809) Measured on the arm as first written: `ref_i8` came up in a ticket (whole family initialised on the host path) and produced **zero logit dumps in four minutes** -- the request never finished a token, because the GEMM quantised every weight column on every call: 128 GEMMs x K*N plus a per-column amax pass is ~7.5 G scalar integer operations for one 7B token. The weights are the same matrix on every call, so they are now quantised **once per tensor (`wkey`)** into `host_i8_cache_` (`codes[K][N]` + per-128-K multipliers `d[groups][N]`), and each call is a pure integer multiply-accumulate through `q4nx_host_gemm_i8_grouped()`, which applies the kernel's own epilogue: `ascale * sum_g d[g][n] * sum_{k in g} code[k][n] * a8[k]`, the inner sum exact int32 with no float in it. The activation row is still quantised per call, which is O(K) and correct to do there. This is what makes the arm usable rather than merely correct: it now measures accumulation + epilogue, which is the half of the gate that "cos 0.9955 against f32" cannot resolve. Test arm 5 pins the grouped form against an independent double computation with the group multipliers applied outside the sum, at K=130 so the group boundary matters. All five arms green; build clean; copyright rc 0. --- npu/forward/include/q4nx_forward.h | 4 ++ npu/forward/include/q4nx_pack.h | 7 +++ npu/forward/src/q4nx_forward.cpp | 69 ++++++++++++++++++++++++++---- npu/forward/src/q4nx_pack.c | 22 ++++++++++ tests/npu_host_gemm_i8_test.cpp | 48 +++++++++++++++++++++ 5 files changed, 142 insertions(+), 8 deletions(-) diff --git a/npu/forward/include/q4nx_forward.h b/npu/forward/include/q4nx_forward.h index 256cab04e..b0a8b2286 100644 --- a/npu/forward/include/q4nx_forward.h +++ b/npu/forward/include/q4nx_forward.h @@ -238,6 +238,10 @@ class Q4nxNpuForward { // construction. It cannot see a wrong weight layout (the f32 arm covers that); it can see a wrong // quantisation, a wrong 128-K group boundary, an int8 intermediate or a saturated accumulator. bool host_gemm_i8_ = false; + // Weight codes + per-128-K multipliers for the integer host arm, quantised once per tensor (engine#1809). + struct HostI8Tensor { int groups = 0; int N = 0; std::vector codes; std::vector d; }; + std::map host_i8_cache_; + void host_gemm_i8(const float* A, int a_k, int M, int K, int N, const float* Bmat, int wkey, float* out); bool load_layer(int l); // `a_k` is the activation's REAL width per row (the design's real_k, i.e. the model's hidden or // intermediate size). It is K whenever the model is already on the container's 256-column K diff --git a/npu/forward/include/q4nx_pack.h b/npu/forward/include/q4nx_pack.h index b69ce526d..6f94b1f4c 100644 --- a/npu/forward/include/q4nx_pack.h +++ b/npu/forward/include/q4nx_pack.h @@ -184,6 +184,13 @@ void q4nx_host_gemm_i8(const int8_t* a8, int K, const int8_t* codes, int N, int3 void q4nx_host_gemm_i8_from_f32(const float* A, int a_k, int M, int K, int N, const float* Bmat, int32_t* out32); +/* The reusable form: `codes` and `d` are quantised once per tensor by the caller (a per-call quantisation of + * the weights costs ~7.5 G scalar operations per 7B token, measured — it never finished a token in a + * 9-minute ticket), and this applies the kernel's epilogue: ascale * sum_g d[g][n] * sum_{k in g} code*a8, + * with the inner sum exact int32 and no float inside it. */ +void q4nx_host_gemm_i8_grouped(const int8_t* a8, int K, const int8_t* codes, const float* d, + int groups, int N, float ascale, float* out); + /* Architecture dims a design set is derived from. */ typedef struct { diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index c4c5b4cf5..821c8ae98 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -3032,6 +3032,65 @@ bool Q4nxNpuForward::run_lm_head_gemv(const std::vector& h, std::vector amax) amax = v; + } + if (!(amax >= 1e-12f)) amax = 1.0f; + t.d[(size_t)g * (size_t)N + (size_t)n] = amax / 127.0f; + const float tis = 127.0f / amax; + for (int k = k0; k < k1; k++) { + float v = Bmat[(size_t)k * (size_t)N + (size_t)n]; + if (!std::isfinite(v)) v = 0.0f; + int q = (int)std::round(v * tis); + q = q > 7 ? 7 : (q < -8 ? -8 : q); + t.codes[(size_t)k * (size_t)N + (size_t)n] = (int8_t)q; + } + } + } + it = host_i8_cache_.emplace(wkey, std::move(t)).first; + } + const HostI8Tensor& t = it->second; + std::vector a8((size_t)K); + for (int m = 0; m < M; m++) { + const float* a = A + (size_t)m * (size_t)a_k; + float amax = 0.0f; + for (int k = 0; k < K; k++) { + const float v = std::fabs(a[k]); + if (std::isfinite(v) && v > amax) amax = v; + } + const float ascale = (amax >= 1e-12f) ? amax / 127.0f : 1.0f; + const float tis = 1.0f / ascale; + for (int k = 0; k < K; k++) { + float v = a[k]; + if (!std::isfinite(v)) v = 0.0f; + int q = (int)std::round(v * tis); + a8[(size_t)k] = (int8_t)(q > 127 ? 127 : (q < -127 ? -127 : q)); + } + q4nx_host_gemm_i8_grouped(a8.data(), K, t.codes.data(), t.d.data(), t.groups, N, ascale, + out + (size_t)m * (size_t)N); + } +} + void Q4nxNpuForward::npu_gemm(int role, const float* A, int K, int N, const float* Bmat, std::vector& out, int wkey, const char* sc, int a_k) { if (host_gemm_) { @@ -3041,11 +3100,7 @@ void Q4nxNpuForward::npu_gemm(int role, const float* A, int K, int N, const floa // branch reading past the caller's buffer at a padded design.) out.assign((size_t)N, 0.0f); if (host_gemm_i8_) { - // int32 arm: quantise both sides here, accumulate exactly, then apply the same two factors the - // kernel's epilogue applies (the activation row scale and the per-128-K weight multiplier). - std::vector acc((size_t)N, 0); - q4nx_host_gemm_i8_from_f32(A, a_k, 1, K, N, Bmat, acc.data()); - for (int n = 0; n < N; n++) out[(size_t)n] = (float)acc[(size_t)n]; + host_gemm_i8(A, a_k, 1, K, N, Bmat, wkey, out.data()); return; } q4nx_host_gemm_f32_padded(A, a_k, 1, K, N, Bmat, out.data()); @@ -3068,9 +3123,7 @@ void Q4nxNpuForward::npu_gemm_rows(int role, const float* A, int M, int K, int N // caller's buffer is laid out at `m * a_k`, so every row past the first read the wrong row. out.assign((size_t)M * N, 0.0f); if (host_gemm_i8_) { - std::vector acc((size_t)M * (size_t)N, 0); - q4nx_host_gemm_i8_from_f32(A, a_k, M, K, N, Bmat, acc.data()); - for (size_t i = 0; i < acc.size(); i++) out[i] = (float)acc[i]; + host_gemm_i8(A, a_k, M, K, N, Bmat, wkey, out.data()); return; } q4nx_host_gemm_f32_padded(A, a_k, M, K, N, Bmat, out.data()); diff --git a/npu/forward/src/q4nx_pack.c b/npu/forward/src/q4nx_pack.c index 696036661..0914232e5 100644 --- a/npu/forward/src/q4nx_pack.c +++ b/npu/forward/src/q4nx_pack.c @@ -153,6 +153,28 @@ void q4nx_pad_a_f32(const float* A, int M, int real_k, int K, float* out) { * boundary, an int8 intermediate or a saturated accumulator will. That is the split the MoE oracle already * uses (`expected_int32` with the scales factored out), and it is what makes an int32 bit-exact comparison * meaningful at all. */ +/* The per-call cost that matters (engine#1809, measured): quantising the WEIGHTS inside the GEMM made one 7B + * token ~7.5 G scalar operations, so the arm never finished a token in a 9-minute ticket. The weights are the + * same matrix on every call, so they are quantised once per tensor by the caller and reuse this form: + * + * out[n] = ascale * sum_g d[g][n] * ( sum_{k in group g} code[k][n] * a8[k] ) + * + * exactly the kernel's epilogue, with the inner sum in int32 and no float in it. `d` is the per-128-K-group + * multiplier per column ([groups][N]), `ascale` the activation row's scale. */ +void q4nx_host_gemm_i8_grouped(const int8_t* a8, int K, const int8_t* codes, const float* d, + int groups, int N, float ascale, float* out) { + const int G = 128; + for (int n = 0; n < N; n++) out[n] = 0.0f; + for (int g = 0; g < groups; g++) { + const int k0 = g * G, k1 = (k0 + G < K) ? k0 + G : K; + for (int n = 0; n < N; n++) { + int32_t acc = 0; + for (int k = k0; k < k1; k++) acc += (int32_t)((int)codes[(size_t)k * (size_t)N + (size_t)n] * (int)a8[k]); + out[n] += d[(size_t)g * (size_t)N + (size_t)n] * (ascale * (float)acc); + } + } +} + void q4nx_host_gemm_i8_from_f32(const float* A, int a_k, int M, int K, int N, const float* Bmat, int32_t* out32) { const int G = 128; /* the arena's multiplier group: one d per 128 K-rows per column */ diff --git a/tests/npu_host_gemm_i8_test.cpp b/tests/npu_host_gemm_i8_test.cpp index 4339b4047..663055a47 100644 --- a/tests/npu_host_gemm_i8_test.cpp +++ b/tests/npu_host_gemm_i8_test.cpp @@ -120,6 +120,54 @@ int main() { check(ok, "whole-GEMM i8 arm: exact int32, 128-K group boundary respected at K=130"); } + // 5. the reusable grouped form the forward now calls: same epilogue as the kernel, inner sum exact int32. + // Checked against an independent double computation with the 128-K group multipliers applied outside. + { + const int KK = 130, NN = 2, groups = 2; + std::vector A(KK), B((size_t)KK * NN), want((size_t)NN, 0.0f); + std::vector codes((size_t)KK * NN, 0); + std::vector d((size_t)groups * NN, 0.0f); + for (int k = 0; k < KK; k++) A[k] = (float)(k % 13) - 6.0f; + for (int k = 0; k < KK; k++) + for (int n = 0; n < NN; n++) B[(size_t)k * NN + n] = (float)((k * 3 + n) % 7) - 3.0f; + for (int g = 0; g < groups; g++) { + const int k0 = g * 128, k1 = (k0 + 128 < KK) ? k0 + 128 : KK; + for (int n = 0; n < NN; n++) { + float amax = 0.0f; + for (int k = k0; k < k1; k++) amax = std::max(amax, std::fabs(B[(size_t)k * NN + n])); + if (amax < 1e-12f) amax = 1.0f; + d[(size_t)g * NN + n] = amax / 127.0f; + const float tis = 127.0f / amax; + for (int k = k0; k < k1; k++) { + int q = (int)std::round(B[(size_t)k * NN + n] * tis); + codes[(size_t)k * NN + n] = (int8_t)(q > 7 ? 7 : (q < -8 ? -8 : q)); + } + } + } + float amax = 0.0f; + for (int k = 0; k < KK; k++) amax = std::max(amax, std::fabs(A[k])); + const float ascale = amax / 127.0f, tis = 1.0f / ascale; + std::vector a8(KK); + for (int k = 0; k < KK; k++) { + int q = (int)std::round(A[k] * tis); + a8[k] = (int8_t)(q > 127 ? 127 : (q < -127 ? -127 : q)); + } + for (int g = 0; g < groups; g++) { + const int k0 = g * 128, k1 = (k0 + 128 < KK) ? k0 + 128 : KK; + for (int n = 0; n < NN; n++) { + double acc = 0; + for (int k = k0; k < k1; k++) acc += (double)codes[(size_t)k * NN + n] * (double)a8[k]; + want[(size_t)n] += (float)(d[(size_t)g * NN + n] * (double)(ascale * (float)acc)); + } + } + std::vector got((size_t)NN, 0.0f); + q4nx_host_gemm_i8_grouped(a8.data(), KK, codes.data(), d.data(), groups, NN, ascale, got.data()); + bool ok = true; + for (int n = 0; n < NN; n++) + if (std::fabs(got[(size_t)n] - want[(size_t)n]) > 1e-5f * std::fabs(want[(size_t)n]) + 1e-6f) ok = false; + check(ok, "grouped form: int32 inner sum + per-128-K multipliers match an independent computation"); + } + if (failures) { std::fprintf(stderr, "%d check(s) failed\n", failures); return 1; } std::printf("all npu_host_gemm_i8 checks passed\n"); return 0; From 852b13ec42820ebdcef185a037a2dd52b2f52278 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 02:33:54 -0400 Subject: [PATCH 06/14] npu: the integer host GEMM's test compiles the sources that define and feed it The build was red on the PR's own gate: undefined reference to `q4nx_host_gemm_i8` `q4nx_host_gemm_i8` is defined in npu/forward/src/q4nx_pack.c, which is compiled into `onebit_npu_forward` -- not into `onebit_npu`, which line 854 links. The file's own convention for exactly this case is a few lines down: `npu_design_grid_test`, `npu_elf_args_test` and `host_gemm_pad_test` all list `npu/forward/src/q4nx_pack.c` as a source. This head's q4nx_pack.c also calls `q4nx_dequant_tensor_chunked`, so the test takes q4nx_dequant.c with it. Verified on this head: `npu_host_gemm_i8_test` builds and all checks pass, including the 128-K group boundary at K=130 and the grouped int32 form against an independent computation. --- CMakeLists.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 81f548b8a..2cb787e92 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -851,7 +851,10 @@ add_test(NAME route_port_lock COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/ # forward offers is f32 -- which cannot be bit-exact with an int8 x int4 device by construction. This pins # the integer arm (q4nx_host_gemm_i8) against an independently generated known answer, including an # activation of -128 so a naive int8 intermediate or an int32 saturation would show. Pure host, no device. - add_executable(npu_host_gemm_i8_test tests/npu_host_gemm_i8_test.cpp) + # q4nx_pack.c is the file that defines q4nx_host_gemm_i8 (it is in onebit_npu_forward, not onebit_npu), and + # its newer body also calls q4nx_dequant_tensor_chunked, so both sources come with the test. + add_executable(npu_host_gemm_i8_test tests/npu_host_gemm_i8_test.cpp + npu/forward/src/q4nx_pack.c npu/forward/src/q4nx_dequant.c) target_include_directories(npu_host_gemm_i8_test PRIVATE npu/forward/include ${CMAKE_SOURCE_DIR}/tests) target_link_libraries(npu_host_gemm_i8_test PRIVATE onebit_npu) add_test(NAME npu_host_gemm_i8 COMMAND npu_host_gemm_i8_test) From 326ff207faeae377ad1f118ba81927f1a348939b Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 02:32:35 -0400 Subject: [PATCH 07/14] ci: build npu_host_gemm_i8_test, so the ctest that pins the integer arm actually runs The new test was registered in CMakeLists.txt but not listed in ci.yml's build targets, so in the configuration CI creates it reported 'Not Run' and the ci_targets check failed with exactly that: "add_test names 'npu_host_gemm_i8_test', and ci.yml never builds npu_host_gemm_i8 -- that test never reaches ctest". Local check now: 31 targets, 9 tests whose target is not built under ci.yml (the baseline), none new. --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b7c8399f8..5a1b501e2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -130,7 +130,8 @@ jobs: npu_model_bounds_test npu_forward_loader_test npu_design_grid_test npu_sampling_test \ npu_flm_head_import_test npu_elf_args_test \ npu_template_test npu_chatml_test \ - tokenizer_linear_test hrx_placement_test hrx_cwsr_test hrx_runtime_test npu_device_lock_test -j + tokenizer_linear_test hrx_placement_test hrx_cwsr_test hrx_runtime_test npu_device_lock_test \ + npu_host_gemm_i8_test -j - name: Test # One hung test must not hold the job for hours: each test gets 10 minutes. run: ctest --test-dir build --output-on-failure --timeout 600 From 49220fb37a031928b239f8552f6a681da4f1b59e Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 02:34:04 -0400 Subject: [PATCH 08/14] ci: fold the integer arm's checks into npu_pack_test, because ci.yml cannot be edited with this token The build was red for two reasons, both from one line: `ci_targets` reported "add_test names 'npu_host_gemm_i8_test', and ci.yml never builds npu_host_gemm_i8", and the test itself reported "Not Run" in the configuration CI creates. The check's own advice is to list the target in ci.yml -- but a push touching `.github/workflows/ci.yml` is rejected for this lane's token ("refusing to allow an OAuth App to create or update workflow `.github/workflows/ci.yml` without `workflow` scope"), and I am not going to work around a token scope. So the checks moved into `tests/npu_pack_test.cpp`, which ci.yml already builds and ctest already runs, next to the pack/tile checks they belong with: - int8 x int4 -> int32 against the independently generated fixture (tools/host_gemm_i8_fixture.py); - every weight code an int4 value in [-8, 7]; - a -128 activation accumulating exactly (no int8 intermediate, no int32 saturation); - the grouped form's int32 inner sum plus per-128-K multipliers against an independent computation at K=130. The separate target, its add_test and its source are gone; the fixture and its generator stay. ci.yml is back to its committed state. `tools/ci/ci_targets_check.py --check` reports the 9-test baseline and nothing new, and `npu_pack_test` builds and passes. --- .github/workflows/ci.yml | 3 +- CMakeLists.txt | 22 ++-- tests/npu_host_gemm_i8_test.cpp | 174 -------------------------------- tests/npu_pack_test.cpp | 74 ++++++++++++++ 4 files changed, 85 insertions(+), 188 deletions(-) delete mode 100644 tests/npu_host_gemm_i8_test.cpp diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a1b501e2..b7c8399f8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -130,8 +130,7 @@ jobs: npu_model_bounds_test npu_forward_loader_test npu_design_grid_test npu_sampling_test \ npu_flm_head_import_test npu_elf_args_test \ npu_template_test npu_chatml_test \ - tokenizer_linear_test hrx_placement_test hrx_cwsr_test hrx_runtime_test npu_device_lock_test \ - npu_host_gemm_i8_test -j + tokenizer_linear_test hrx_placement_test hrx_cwsr_test hrx_runtime_test npu_device_lock_test -j - name: Test # One hung test must not hold the job for hours: each test gets 10 minutes. run: ctest --test-dir build --output-on-failure --timeout 600 diff --git a/CMakeLists.txt b/CMakeLists.txt index 2cb787e92..ed9db049b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -847,17 +847,10 @@ add_test(NAME route_port_lock COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/ # the whole run loop against canned completions, and re-derives the summary from the written transcript. Stdlib # only, with no route, no socket, no dataset fetch and no device. add_test(NAME gsm8k_eval COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/gsm8k_eval_test.py) - # engine#1808: the kernel gate asks for "bit-exact against the host reference", but the only host arm the - # forward offers is f32 -- which cannot be bit-exact with an int8 x int4 device by construction. This pins - # the integer arm (q4nx_host_gemm_i8) against an independently generated known answer, including an - # activation of -128 so a naive int8 intermediate or an int32 saturation would show. Pure host, no device. - # q4nx_pack.c is the file that defines q4nx_host_gemm_i8 (it is in onebit_npu_forward, not onebit_npu), and - # its newer body also calls q4nx_dequant_tensor_chunked, so both sources come with the test. - add_executable(npu_host_gemm_i8_test tests/npu_host_gemm_i8_test.cpp - npu/forward/src/q4nx_pack.c npu/forward/src/q4nx_dequant.c) - target_include_directories(npu_host_gemm_i8_test PRIVATE npu/forward/include ${CMAKE_SOURCE_DIR}/tests) - target_link_libraries(npu_host_gemm_i8_test PRIVATE onebit_npu) - add_test(NAME npu_host_gemm_i8 COMMAND npu_host_gemm_i8_test) + # engine#1808's integer host arm is pinned inside tests/npu_pack_test.cpp (ctest npu_pack): that binary is + # already built by ci.yml, and a new ctest target would need a `.github/workflows/ci.yml` edit, which this + # lane's token cannot push (the OAuth app has no `workflow` scope) -- the ci_targets check caught exactly + # that, with both failures ("ci_targets" and "npu_host_gemm_i8 Not Run"). # The repack's tile-order claim (engine#1110): `npu_plan.py` writes container.tile_order=engine-1 for every # plan, so the load-time guard (#1130) is green on a container whose TILES are another order. This pins the # mapping from q4nx_diff's verdict to (stamp value, fatal), stdlib only: no model, no numpy, no device. @@ -1106,13 +1099,18 @@ if(ONEBIT_OGA_PACK) add_test(NAME npu_pack_import_oga COMMAND npu_pack_import_test ${CMAKE_SOURCE_DIR}/tests/golden ${ONEBIT_OGA_PACK}) endif() -add_executable(npu_pack_test tests/npu_pack_test.cpp) +# q4nx_pack.c is listed here rather than reached through onebit_npu: the integer host arm's checks +# (engine#1808) call q4nx_host_gemm_i8*() directly, and those live in that file. +add_executable(npu_pack_test tests/npu_pack_test.cpp npu/forward/src/q4nx_pack.c npu/forward/src/q4nx_dequant.c) add_executable(flm_head_import tools/flm_head_import.cpp) target_link_libraries(flm_head_import PRIVATE onebit_npu) add_executable(npu_flm_head_import_test tests/npu_flm_head_import_test.cpp) target_link_libraries(npu_flm_head_import_test PRIVATE onebit_npu) add_test(NAME npu_flm_head_import COMMAND npu_flm_head_import_test ${CMAKE_SOURCE_DIR}/tests/golden/flm-i8) target_link_libraries(npu_pack_test PRIVATE onebit_npu) +# engine#1808: the integer host arm's checks live in this binary (see above), so it needs the header and the +# golden fixture that pins them. +target_include_directories(npu_pack_test PRIVATE npu/forward/include ${CMAKE_SOURCE_DIR}/tests) # The device weight layout contract (q4nx_weight_layout): pure index math, no device # and no runtime. CI builds an explicit target list, so the check compiles itself with diff --git a/tests/npu_host_gemm_i8_test.cpp b/tests/npu_host_gemm_i8_test.cpp deleted file mode 100644 index 663055a47..000000000 --- a/tests/npu_host_gemm_i8_test.cpp +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright 2026 bong-water-water-bong -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// npu_host_gemm_i8_test: the known answer for the integer host arm (engine#1808). -// -// The gate for a kernel family asks for "bit-exact against the host reference". The only host arm the -// forward offers is f32, and f32 cannot be bit-exact with an int8 x int4 device by construction, so the -// comparison can only ever produce a quality number. `q4nx_host_gemm_i8()` is the missing arm: exact int32 -// accumulation of the same operands the kernel gets. -// -// This test is a *known answer*, not a re-implementation: the fixture in tests/golden/ was generated by an -// independent Python computation (tools/host_gemm_i8_fixture.py) with a fixed seed, so a change that -// "agrees with itself" cannot pass. Three arms: -// 1. the fixture: K=128, N=4, int8 activations and int4 codes, expected int32 per column; -// 2. the int4 range: every code in the fixture lies in [-8, 7] (a code outside it would mean the caller -// is handing us a q4_1 byte instead of a code, which is the mistake the whole gate exists to catch); -// 3. the edge: an activation of -128 (outside the quantiser's range, inside int8) must still accumulate -// exactly rather than saturate at the int32 level -- the product bound is 128 * 8 * K, which for the -// largest K in use (14336) is 1.47e7, far inside int32, so a saturation here would be a real bug. - -#include "q4nx_pack.h" - -#include -#include -#include -#include -#include - -#include "golden/host_gemm_i8_fixture.h" - -static int failures = 0; - -static void check(bool ok, const char* what) { - std::printf("%s %s\n", ok ? "ok " : "FAIL", what); - if (!ok) failures++; -} - -int main() { - // 1. the recorded known answer - int32_t got[kHostI8N] = {0, 0, 0, 0}; - q4nx_host_gemm_i8(kHostI8A, kHostI8K, &kHostI8Codes[0][0], kHostI8N, got); - bool same = true; - for (int n = 0; n < kHostI8N; n++) { - if (got[n] != kHostI8Expected[n]) { - std::printf(" column %d: got %d, expected %d\n", n, (int)got[n], (int)kHostI8Expected[n]); - same = false; - } - } - check(same, "int8 x int4 -> int32 matches the independent Python fixture, all 4 columns"); - - // 2. the fixture only carries int4 codes - bool in_range = true; - for (int k = 0; k < kHostI8K && in_range; k++) - for (int n = 0; n < kHostI8N; n++) - if (kHostI8Codes[k][n] < -8 || kHostI8Codes[k][n] > 7) in_range = false; - check(in_range, "every weight code is an int4 value in [-8, 7]"); - - // 3. the int32 edge: a naive int8 multiply would overflow at 128 * 127, this must not - const int K = 256; - static int8_t a8[256]; - static int8_t codes[256 * 2]; - for (int k = 0; k < K; k++) { a8[k] = (k == 0) ? -128 : 127; codes[k * 2] = -8; codes[k * 2 + 1] = 7; } - int32_t edge[2] = {0, 0}; - q4nx_host_gemm_i8(a8, K, codes, 2, edge); - // column 0: -8 * (-128) + -8 * 127 * 255 = 1024 - 259080 = -258056 - // column 1: 7 * (-128) + 7 * 127 * 255 = -896 + 226695 = 225799 - check(edge[0] == -258056 && edge[1] == 225799, - "a -128 activation accumulates exactly (no int8 intermediate, no int32 saturation)"); - - // 4. the whole-GEMM arm the gate switches on: quantise both sides here, accumulate exactly, and check - // the two properties that make the comparison meaningful -- the per-128-K group boundary is honoured - // (a K of 130 must not fold the last two rows into the first group's scale) and int4 codes stay in - // [-8, 7] so an out-of-range code cannot silently wrap. - { - const int KK = 130, NN = 3, MM = 2; - std::vector A((size_t)MM * KK), B((size_t)KK * NN); - for (int m = 0; m < MM; m++) - for (int k = 0; k < KK; k++) A[(size_t)m * KK + k] = (float)((k * 7 + m * 3) % 11) - 5.0f; - for (int k = 0; k < KK; k++) - for (int n = 0; n < NN; n++) B[(size_t)k * NN + n] = (float)((k * 5 + n * 13) % 17) / 3.0f; - std::vector out((size_t)MM * NN, 0); - q4nx_host_gemm_i8_from_f32(A.data(), KK, MM, KK, NN, B.data(), out.data()); - // independent computation in long double, from the same quantisation rule - bool ok = true; - for (int m = 0; m < MM && ok; m++) { - for (int n = 0; n < NN && ok; n++) { - double want = 0; - for (int g = 0; g < 2; g++) { // 128 + 2 rows = two groups - const int k0 = g * 128, k1 = (k0 + 128 < KK) ? k0 + 128 : KK; - double amax = 0; - for (int k = k0; k < k1; k++) amax = std::max(amax, std::fabs((double)B[(size_t)k * NN + n])); - if (amax < 1e-12) amax = 1; - const double tis = 127.0 / amax; - for (int k = k0; k < k1; k++) { - int q = (int)std::round((double)B[(size_t)k * NN + n] * tis); - q = q > 7 ? 7 : (q < -8 ? -8 : q); - int aq = (int)std::round((double)A[(size_t)m * KK + k] * 127.0); - aq = aq > 127 ? 127 : (aq < -127 ? -127 : aq); - want += (double)q * (double)aq; - } - } - if ((double)out[(size_t)m * NN + n] != want) { - std::printf(" m=%d n=%d: got %d, want %.0f\n", m, n, (int)out[(size_t)m * NN + n], want); - ok = false; - } - } - } - check(ok, "whole-GEMM i8 arm: exact int32, 128-K group boundary respected at K=130"); - } - - // 5. the reusable grouped form the forward now calls: same epilogue as the kernel, inner sum exact int32. - // Checked against an independent double computation with the 128-K group multipliers applied outside. - { - const int KK = 130, NN = 2, groups = 2; - std::vector A(KK), B((size_t)KK * NN), want((size_t)NN, 0.0f); - std::vector codes((size_t)KK * NN, 0); - std::vector d((size_t)groups * NN, 0.0f); - for (int k = 0; k < KK; k++) A[k] = (float)(k % 13) - 6.0f; - for (int k = 0; k < KK; k++) - for (int n = 0; n < NN; n++) B[(size_t)k * NN + n] = (float)((k * 3 + n) % 7) - 3.0f; - for (int g = 0; g < groups; g++) { - const int k0 = g * 128, k1 = (k0 + 128 < KK) ? k0 + 128 : KK; - for (int n = 0; n < NN; n++) { - float amax = 0.0f; - for (int k = k0; k < k1; k++) amax = std::max(amax, std::fabs(B[(size_t)k * NN + n])); - if (amax < 1e-12f) amax = 1.0f; - d[(size_t)g * NN + n] = amax / 127.0f; - const float tis = 127.0f / amax; - for (int k = k0; k < k1; k++) { - int q = (int)std::round(B[(size_t)k * NN + n] * tis); - codes[(size_t)k * NN + n] = (int8_t)(q > 7 ? 7 : (q < -8 ? -8 : q)); - } - } - } - float amax = 0.0f; - for (int k = 0; k < KK; k++) amax = std::max(amax, std::fabs(A[k])); - const float ascale = amax / 127.0f, tis = 1.0f / ascale; - std::vector a8(KK); - for (int k = 0; k < KK; k++) { - int q = (int)std::round(A[k] * tis); - a8[k] = (int8_t)(q > 127 ? 127 : (q < -127 ? -127 : q)); - } - for (int g = 0; g < groups; g++) { - const int k0 = g * 128, k1 = (k0 + 128 < KK) ? k0 + 128 : KK; - for (int n = 0; n < NN; n++) { - double acc = 0; - for (int k = k0; k < k1; k++) acc += (double)codes[(size_t)k * NN + n] * (double)a8[k]; - want[(size_t)n] += (float)(d[(size_t)g * NN + n] * (double)(ascale * (float)acc)); - } - } - std::vector got((size_t)NN, 0.0f); - q4nx_host_gemm_i8_grouped(a8.data(), KK, codes.data(), d.data(), groups, NN, ascale, got.data()); - bool ok = true; - for (int n = 0; n < NN; n++) - if (std::fabs(got[(size_t)n] - want[(size_t)n]) > 1e-5f * std::fabs(want[(size_t)n]) + 1e-6f) ok = false; - check(ok, "grouped form: int32 inner sum + per-128-K multipliers match an independent computation"); - } - - if (failures) { std::fprintf(stderr, "%d check(s) failed\n", failures); return 1; } - std::printf("all npu_host_gemm_i8 checks passed\n"); - return 0; -} diff --git a/tests/npu_pack_test.cpp b/tests/npu_pack_test.cpp index 6ff6ad82a..905c1d411 100644 --- a/tests/npu_pack_test.cpp +++ b/tests/npu_pack_test.cpp @@ -22,6 +22,11 @@ // i5_.bin, i6_.bin for layers 0 and 2). #include "full_elf.h" #include "pack.h" +#include "q4nx_pack.h" + +#include "golden/host_gemm_i8_fixture.h" + +#include #include #include @@ -62,6 +67,75 @@ int main(int argc, char** argv) { check(f32_to_bf16(1.0f) == 0x3F80, "bf16(1.0)"); check(f32_to_bf16(1.00390625f) == 0x3F80, "bf16 ties to even (down)"); check(f32_to_bf16(1.01171875f) == 0x3F82, "bf16 ties to even (up)"); + // The integer host arm (engine#1808/#1809): "bit-exact against the host reference" needs a host path that + // runs the SAME integer arithmetic as the kernel, and f32 cannot be bit-exact with int8 x int4 by + // construction. These checks live here, next to the pack/tile checks they belong with, because ci.yml + // builds this binary and a new ctest target would need a workflow edit this lane's token cannot make. + { + int32_t got[kHostI8N] = {0, 0, 0, 0}; + q4nx_host_gemm_i8(kHostI8A, kHostI8K, &kHostI8Codes[0][0], kHostI8N, got); + bool same = true; + for (int n = 0; n < kHostI8N; n++) same &= got[n] == kHostI8Expected[n]; + check(same, "int8 x int4 -> int32 matches the independent Python fixture (golden/host_gemm_i8_fixture.h)"); + bool in_range = true; + for (int k = 0; k < kHostI8K; k++) + for (int n = 0; n < kHostI8N; n++) + if (kHostI8Codes[k][n] < -8 || kHostI8Codes[k][n] > 7) in_range = false; + check(in_range, "every weight code is an int4 value in [-8, 7]"); + // a -128 activation is outside the quantiser's range but inside int8: it must accumulate exactly, and + // the product bound for the largest K in use (14336) is 1.47e7, so int32 saturation would be a bug + static int8_t a8[256], codes2[512]; + for (int k = 0; k < 256; k++) { a8[k] = (k == 0) ? -128 : 127; codes2[k * 2] = -8; codes2[k * 2 + 1] = 7; } + int32_t edge[2] = {0, 0}; + q4nx_host_gemm_i8(a8, 256, codes2, 2, edge); + check(edge[0] == -258056 && edge[1] == 225799, + "a -128 activation accumulates exactly (no int8 intermediate, no int32 saturation)"); + // the reusable grouped form: inner sum int32, per-128-K multipliers applied outside it, K=130 so the + // group boundary matters + const int KK = 130, NN = 2, groups = 2; + std::vector A(KK), B((size_t)KK * NN); + std::vector codes((size_t)KK * NN, 0); + std::vector d((size_t)groups * NN, 0.0f), want((size_t)NN, 0.0f); + for (int k = 0; k < KK; k++) A[k] = float(k % 13) - 6.0f; + for (int k = 0; k < KK; k++) + for (int n = 0; n < NN; n++) B[(size_t)k * NN + n] = float((k * 3 + n) % 7) - 3.0f; + float amax_all = 0.0f; + for (int k = 0; k < KK; k++) amax_all = std::max(amax_all, std::fabs(A[k])); + const float ascale = amax_all / 127.0f, tis = 1.0f / ascale; + std::vector a8q(KK); + for (int k = 0; k < KK; k++) { + int q = int(std::round(A[k] * tis)); + a8q[k] = int8_t(q > 127 ? 127 : (q < -127 ? -127 : q)); + } + for (int g = 0; g < groups; g++) { + const int k0 = g * 128, k1 = (k0 + 128 < KK) ? k0 + 128 : KK; + for (int n = 0; n < NN; n++) { + float amax = 0.0f; + for (int k = k0; k < k1; k++) amax = std::max(amax, std::fabs(B[(size_t)k * NN + n])); + if (amax < 1e-12f) amax = 1.0f; + d[(size_t)g * NN + n] = amax / 127.0f; + for (int k = k0; k < k1; k++) { + int q = int(std::round(B[(size_t)k * NN + n] * (127.0f / amax))); + codes[(size_t)k * NN + n] = int8_t(q > 7 ? 7 : (q < -8 ? -8 : q)); + } + } + } + for (int g = 0; g < groups; g++) { + const int k0 = g * 128, k1 = (k0 + 128 < KK) ? k0 + 128 : KK; + for (int n = 0; n < NN; n++) { + double acc = 0; + for (int k = k0; k < k1; k++) acc += double(codes[(size_t)k * NN + n]) * double(a8q[k]); + want[(size_t)n] += float(d[(size_t)g * NN + n] * double(ascale * float(acc))); + } + } + std::vector outv((size_t)NN, 0.0f); + q4nx_host_gemm_i8_grouped(a8q.data(), KK, codes.data(), d.data(), groups, NN, ascale, outv.data()); + bool ok = true; + for (int n = 0; n < NN; n++) + ok &= std::fabs(outv[(size_t)n] - want[(size_t)n]) <= 1e-5f * std::fabs(want[(size_t)n]) + 1e-6f; + check(ok, "grouped form: int32 inner sum + per-128-K multipliers match an independent computation"); + } + if (argc == 1) { std::printf("%s\n", failures ? "FAILED" : "ok (pass for the model checks)"); return failures ? 1 : 0; From d3e8bd5f4d10ad779a2d1dd4bad59b008c45851d Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 03:05:04 -0400 Subject: [PATCH 09/14] npu: the integer arm quantises weights the way our own packer does, not with a q4_1 convention The arm computed, and computed a DIFFERENT model: it re-quantised the dequantised f32 matrix to [-8, 7] per 128-K group, while our own `packB_blocked()` gives a weight column ONE multiplier over its whole K with int8 codes in [-127, 127]. Measured consequence: the host arm answered "ikt" to "The capital of France is" and then locked onto a single token, which is what produced the flat top-5 in the gate table -- not a dead GEMM (checked with a host-only debug build: codes, multipliers, activation codes and outputs were all populated and varying). `q4nx_host_gemm_i8_col()` reproduces OUR rule -- one multiplier per column, int32 inner sum with no float inside -- and the forward's cache now fills with it. The per-128-K int4 form stays only as an arithmetic fixture, because it is a different numeric model and can never be a bit-exactness reference for these packs. Also: the cache is keyed by (wkey, K, N) rather than `wkey` alone, so two tensors that share a key but not a shape cannot reuse each other's codes. Measured cost, which is the next thing to fix: the arm's first pass costs a **173.9 s** warm-up on this 7B container, because the quantiser is scalar and runs per tensor. Correct, and too slow to iterate against -- the engine's own packed path is the thing to reuse next, rather than a faster copy of it. Test: the ctest arm that used to pin the int4 rule now pins the column rule against an independent double computation; `npu_pack_test` builds and passes; copyright rc 0; ci_targets baseline unchanged. --- npu/forward/include/q4nx_forward.h | 5 +- npu/forward/include/q4nx_pack.h | 7 +++ npu/forward/src/q4nx_forward.cpp | 52 +++++++++++---------- npu/forward/src/q4nx_pack.c | 17 +++++++ tests/npu_pack_test.cpp | 74 +++++++++++++----------------- 5 files changed, 89 insertions(+), 66 deletions(-) diff --git a/npu/forward/include/q4nx_forward.h b/npu/forward/include/q4nx_forward.h index b0a8b2286..590338f80 100644 --- a/npu/forward/include/q4nx_forward.h +++ b/npu/forward/include/q4nx_forward.h @@ -36,6 +36,7 @@ #include #include #include +#include #include "moe_container.h" // the two MoE container conventions (engine#1035) #include "model.h" @@ -240,7 +241,9 @@ class Q4nxNpuForward { bool host_gemm_i8_ = false; // Weight codes + per-128-K multipliers for the integer host arm, quantised once per tensor (engine#1809). struct HostI8Tensor { int groups = 0; int N = 0; std::vector codes; std::vector d; }; - std::map host_i8_cache_; + // Keyed by (wkey, K, N): a wkey alone is not enough -- a second tensor with the same key and a + // different shape would reuse the first one's codes. + std::map, HostI8Tensor> host_i8_cache_; void host_gemm_i8(const float* A, int a_k, int M, int K, int N, const float* Bmat, int wkey, float* out); bool load_layer(int l); // `a_k` is the activation's REAL width per row (the design's real_k, i.e. the model's hidden or diff --git a/npu/forward/include/q4nx_pack.h b/npu/forward/include/q4nx_pack.h index 6f94b1f4c..373e113c3 100644 --- a/npu/forward/include/q4nx_pack.h +++ b/npu/forward/include/q4nx_pack.h @@ -191,6 +191,13 @@ void q4nx_host_gemm_i8_from_f32(const float* A, int a_k, int M, int K, int N, co void q4nx_host_gemm_i8_grouped(const int8_t* a8, int K, const int8_t* codes, const float* d, int groups, int N, float ascale, float* out); +/* The form that matches `packB_blocked()`: one multiplier per weight COLUMN (amax over the whole K, codes int8 + * in [-127,127]), inner sum exact int32. This is the arm a bit-exactness gate can use, because the operands are + * the ones our own packing produces; the per-128-K int4 form above re-quantises into a different model and is + * kept only for the arithmetic tests. */ +void q4nx_host_gemm_i8_col(const int8_t* a8, int K, const int8_t* codes, const float* d, int N, float ascale, + float* out); + /* Architecture dims a design set is derived from. */ typedef struct { diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 821c8ae98..8816c8dad 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -48,6 +48,8 @@ #include "q4nx_semantics.h" #include "q4nx_pack.h" +#include + #if defined(_OPENMP) #include #endif @@ -3041,33 +3043,35 @@ bool Q4nxNpuForward::run_lm_head_gemv(const std::vector& h, std::vector amax) amax = v; - } - if (!(amax >= 1e-12f)) amax = 1.0f; - t.d[(size_t)g * (size_t)N + (size_t)n] = amax / 127.0f; - const float tis = 127.0f / amax; - for (int k = k0; k < k1; k++) { - float v = Bmat[(size_t)k * (size_t)N + (size_t)n]; - if (!std::isfinite(v)) v = 0.0f; - int q = (int)std::round(v * tis); - q = q > 7 ? 7 : (q < -8 ? -8 : q); - t.codes[(size_t)k * (size_t)N + (size_t)n] = (int8_t)q; - } + t.d.assign((size_t)N, 0.0f); + // The rule packB_blocked() applies to the weights the device streams: ONE multiplier per column, amax + // over the whole K, codes in [-127, 127]. Anything else is a different numeric model and cannot be + // compared bit for bit (measured 2026-11-10: a per-128-K int4 re-quantisation answered "ikt" and then + // locked onto a single token). + for (int n = 0; n < N; n++) { + float amax = 0.0f; + for (int k = 0; k < K; k++) { + const float v = std::fabs(Bmat[(size_t)k * (size_t)N + (size_t)n]); + if (std::isfinite(v) && v > amax) amax = v; + } + if (!(amax >= 1e-12f)) amax = 1.0f; + t.d[(size_t)n] = amax / 127.0f; + const float tis = 127.0f / amax; + for (int k = 0; k < K; k++) { + float v = Bmat[(size_t)k * (size_t)N + (size_t)n]; + if (!std::isfinite(v)) v = 0.0f; + int q = (int)std::round(v * tis); + q = q > 127 ? 127 : (q < -127 ? -127 : q); + t.codes[(size_t)k * (size_t)N + (size_t)n] = (int8_t)q; } } - it = host_i8_cache_.emplace(wkey, std::move(t)).first; + it = host_i8_cache_.emplace(key, std::move(t)).first; } const HostI8Tensor& t = it->second; std::vector a8((size_t)K); @@ -3086,8 +3090,8 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N, int q = (int)std::round(v * tis); a8[(size_t)k] = (int8_t)(q > 127 ? 127 : (q < -127 ? -127 : q)); } - q4nx_host_gemm_i8_grouped(a8.data(), K, t.codes.data(), t.d.data(), t.groups, N, ascale, - out + (size_t)m * (size_t)N); + q4nx_host_gemm_i8_col(a8.data(), K, t.codes.data(), t.d.data(), N, ascale, + out + (size_t)m * (size_t)N); } } diff --git a/npu/forward/src/q4nx_pack.c b/npu/forward/src/q4nx_pack.c index 0914232e5..e29b53ced 100644 --- a/npu/forward/src/q4nx_pack.c +++ b/npu/forward/src/q4nx_pack.c @@ -161,6 +161,23 @@ void q4nx_pad_a_f32(const float* A, int M, int real_k, int K, float* out) { * * exactly the kernel's epilogue, with the inner sum in int32 and no float in it. `d` is the per-128-K-group * multiplier per column ([groups][N]), `ascale` the activation row's scale. */ +/* The form that matches OUR packing, not a q4_1 convention. `packB_blocked()` (engine-src) quantises a weight + * column with ONE multiplier over its whole K -- amax over the column, d = amax/127, codes in [-127, 127] as + * int8 -- so that is the arithmetic a bit-exactness gate has to reproduce. The earlier per-128-K int4 form + * re-quantised the container into a different numeric model (measured: the host arm answered 'ikt' and then + * locked onto one token), which is why it could not be a reference. + * + * out[n] = d[n] * ascale * sum_k code[k][n] * a8[k], inner sum exact int32, no float inside it. */ +void q4nx_host_gemm_i8_col(const int8_t* a8, int K, const int8_t* codes, const float* d, int N, float ascale, + float* out) { + for (int n = 0; n < N; n++) { + int32_t acc = 0; + for (int k = 0; k < K; k++) + acc += (int32_t)((int)codes[(size_t)k * (size_t)N + (size_t)n] * (int)a8[k]); + out[n] = d[n] * (ascale * (float)acc); + } +} + void q4nx_host_gemm_i8_grouped(const int8_t* a8, int K, const int8_t* codes, const float* d, int groups, int N, float ascale, float* out) { const int G = 128; diff --git a/tests/npu_pack_test.cpp b/tests/npu_pack_test.cpp index 905c1d411..a6bcae9b5 100644 --- a/tests/npu_pack_test.cpp +++ b/tests/npu_pack_test.cpp @@ -90,50 +90,42 @@ int main(int argc, char** argv) { q4nx_host_gemm_i8(a8, 256, codes2, 2, edge); check(edge[0] == -258056 && edge[1] == 225799, "a -128 activation accumulates exactly (no int8 intermediate, no int32 saturation)"); - // the reusable grouped form: inner sum int32, per-128-K multipliers applied outside it, K=130 so the - // group boundary matters - const int KK = 130, NN = 2, groups = 2; - std::vector A(KK), B((size_t)KK * NN); - std::vector codes((size_t)KK * NN, 0); - std::vector d((size_t)groups * NN, 0.0f), want((size_t)NN, 0.0f); - for (int k = 0; k < KK; k++) A[k] = float(k % 13) - 6.0f; - for (int k = 0; k < KK; k++) - for (int n = 0; n < NN; n++) B[(size_t)k * NN + n] = float((k * 3 + n) % 7) - 3.0f; - float amax_all = 0.0f; - for (int k = 0; k < KK; k++) amax_all = std::max(amax_all, std::fabs(A[k])); - const float ascale = amax_all / 127.0f, tis = 1.0f / ascale; - std::vector a8q(KK); - for (int k = 0; k < KK; k++) { - int q = int(std::round(A[k] * tis)); - a8q[k] = int8_t(q > 127 ? 127 : (q < -127 ? -127 : q)); + // the column form that matches packB_blocked(): ONE multiplier per weight column, amax over the whole + // K, int8 codes, inner sum exact int32. Checked against an independent double computation. + const int KC = 130, NC = 3; + std::vector A2(KC), B2((size_t)KC * NC), want2((size_t)NC, 0.0f); + std::vector codes3((size_t)KC * NC, 0); + std::vector d2((size_t)NC, 0.0f); + for (int k = 0; k < KC; k++) A2[k] = float((k * 7) % 11) - 5.0f; + for (int k = 0; k < KC; k++) + for (int n = 0; n < NC; n++) B2[(size_t)k * NC + n] = float((k * 5 + n * 13) % 17) / 3.0f; + float amaxA = 0.0f; + for (int k = 0; k < KC; k++) amaxA = std::max(amaxA, std::fabs(A2[k])); + const float ascale2 = amaxA / 127.0f, tis2 = 1.0f / ascale2; + std::vector a8c(KC); + for (int k = 0; k < KC; k++) { + int q = int(std::round(A2[k] * tis2)); + a8c[k] = int8_t(q > 127 ? 127 : (q < -127 ? -127 : q)); } - for (int g = 0; g < groups; g++) { - const int k0 = g * 128, k1 = (k0 + 128 < KK) ? k0 + 128 : KK; - for (int n = 0; n < NN; n++) { - float amax = 0.0f; - for (int k = k0; k < k1; k++) amax = std::max(amax, std::fabs(B[(size_t)k * NN + n])); - if (amax < 1e-12f) amax = 1.0f; - d[(size_t)g * NN + n] = amax / 127.0f; - for (int k = k0; k < k1; k++) { - int q = int(std::round(B[(size_t)k * NN + n] * (127.0f / amax))); - codes[(size_t)k * NN + n] = int8_t(q > 7 ? 7 : (q < -8 ? -8 : q)); - } + for (int n = 0; n < NC; n++) { + float amax = 0.0f; + for (int k = 0; k < KC; k++) amax = std::max(amax, std::fabs(B2[(size_t)k * NC + n])); + if (amax < 1e-12f) amax = 1.0f; + d2[(size_t)n] = amax / 127.0f; + for (int k = 0; k < KC; k++) { + int q = int(std::round(B2[(size_t)k * NC + n] * (127.0f / amax))); + codes3[(size_t)k * NC + n] = int8_t(q > 127 ? 127 : (q < -127 ? -127 : q)); } + double acc = 0; + for (int k = 0; k < KC; k++) acc += double(codes3[(size_t)k * NC + n]) * double(a8c[k]); + want2[(size_t)n] = float(d2[(size_t)n] * double(ascale2 * float(acc))); } - for (int g = 0; g < groups; g++) { - const int k0 = g * 128, k1 = (k0 + 128 < KK) ? k0 + 128 : KK; - for (int n = 0; n < NN; n++) { - double acc = 0; - for (int k = k0; k < k1; k++) acc += double(codes[(size_t)k * NN + n]) * double(a8q[k]); - want[(size_t)n] += float(d[(size_t)g * NN + n] * double(ascale * float(acc))); - } - } - std::vector outv((size_t)NN, 0.0f); - q4nx_host_gemm_i8_grouped(a8q.data(), KK, codes.data(), d.data(), groups, NN, ascale, outv.data()); - bool ok = true; - for (int n = 0; n < NN; n++) - ok &= std::fabs(outv[(size_t)n] - want[(size_t)n]) <= 1e-5f * std::fabs(want[(size_t)n]) + 1e-6f; - check(ok, "grouped form: int32 inner sum + per-128-K multipliers match an independent computation"); + std::vector got2((size_t)NC, 0.0f); + q4nx_host_gemm_i8_col(a8c.data(), KC, codes3.data(), d2.data(), NC, ascale2, got2.data()); + bool ok2 = true; + for (int n = 0; n < NC; n++) + ok2 &= std::fabs(got2[(size_t)n] - want2[(size_t)n]) <= 1e-5f * std::fabs(want2[(size_t)n]) + 1e-6f; + check(ok2, "column form (our packing rule): one multiplier per column, int32 inner sum, matches an independent computation"); } if (argc == 1) { From f34ffa174a679e7baed398fdf603a71a0fc19c92 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 03:14:21 -0400 Subject: [PATCH 10/14] npu: the integer arm's weight image comes from the engine's own packer (5.4x faster, still not a reference) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two changes, one measured improvement and one honest status: * `host_gemm_i8()` now calls `q4nx_pack_B_int8_blocked()` -- the SAME host-side packer the device path uses -- instead of quantising the matrix with a rule of its own, and walks the blocked image with the engine's `q4nx_layout::blocked_index()`. Operands are therefore the engine's, by construction, which is what a bit-exactness gate needs. * Measured effect on the 7B container: the first pass went from **173.9 s to 32.2 s** (5.4x). The engine's packer is the fast path; my scalar re-quantisation was not. **Still not a reference, stated plainly:** with the operands now correct the arm answers `'▁/******/▁/******/…'` for "The capital of France is" -- degenerate, in a different way than before but still degenerate. So something outside the weight quantisation is wrong (the activation quantisation, the scales' application, or where the arm is reached from), and no family row may be gated with `NPU_INFER_HOST_GEMM=i8` yet. The next diagnostic is an A/B of ONE GEMM (f32 vs i8, same A, same tensor) rather than another whole-model run: that separates a layout/scale bug from an integration bug in one step. The earlier note that the per-128-K int4 form "re-quantised into a different model" stands as the reason the first version locked onto a token; this commit removes that re-quantisation entirely. --- npu/forward/src/q4nx_forward.cpp | 43 +++++++++++++------------------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 8816c8dad..0d1818552 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -47,6 +47,7 @@ #include "q4nx_device.h" // an imported q4nx-2 container's device form (engine#1185) #include "q4nx_semantics.h" #include "q4nx_pack.h" +#include "q4nx_weight_layout.h" #include @@ -3042,35 +3043,22 @@ bool Q4nxNpuForward::run_lm_head_gemv(const std::vector& h, std::vector amax) amax = v; - } - if (!(amax >= 1e-12f)) amax = 1.0f; - t.d[(size_t)n] = amax / 127.0f; - const float tis = 127.0f / amax; - for (int k = 0; k < K; k++) { - float v = Bmat[(size_t)k * (size_t)N + (size_t)n]; - if (!std::isfinite(v)) v = 0.0f; - int q = (int)std::round(v * tis); - q = q > 127 ? 127 : (q < -127 ? -127 : q); - t.codes[(size_t)k * (size_t)N + (size_t)n] = (int8_t)q; - } - } + double ssum = 0.0; + q4nx_pack_B_int8_blocked(t.codes.data(), Bmat, K, N, K, N, /*blocked=*/1, t.d.data(), &ssum); it = host_i8_cache_.emplace(key, std::move(t)).first; } const HostI8Tensor& t = it->second; @@ -3090,8 +3078,13 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N, int q = (int)std::round(v * tis); a8[(size_t)k] = (int8_t)(q > 127 ? 127 : (q < -127 ? -127 : q)); } - q4nx_host_gemm_i8_col(a8.data(), K, t.codes.data(), t.d.data(), N, ascale, - out + (size_t)m * (size_t)N); + float* dst = out + (size_t)m * (size_t)N; + for (int n = 0; n < N; n++) { + int32_t acc = 0; + for (int k = 0; k < K; k++) + acc += (int32_t)((int)t.codes[q4nx_layout::blocked_index((size_t)k, (size_t)n, K, N)] * (int)a8[(size_t)k]); + dst[(size_t)n] = t.d[(size_t)n] * (ascale * (float)acc); + } } } From e3d60315e24d05f64faf6b863527341d74deb88b Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 03:33:17 -0400 Subject: [PATCH 11/14] npu: the integer arm's GEMMs match f32 at cos 0.9993-0.99999, so the degenerate output is an integration issue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added an env-gated one-GEMM A/B (`NPU_DBG_I8_AB`): for the first few calls it computes the same activation against the same tensor both ways and prints cos/max|d|. On the Mistral-7B 1.9 container, host-only, no device: wkey=0 K=4096 N=6144 cos=0.999991 max|d|=0.0744 (QKV) wkey=1 K=4096 N=4096 cos=0.999868 max|d|=0.000214 (O) wkey=2 K=4096 N=28672 cos=0.999808 max|d|=0.0227 (GU) wkey=4 K=14336 N=4096 cos=0.999261 max|d|=0.00212 (D) So the weight image, the blocked layout, the per-column scales and the activation quantisation are all correct: the arm computes the GEMM it is supposed to. The `'▁/******/▁/******/'` output therefore comes from how the arm is reached, not from the arithmetic -- the next diagnostic is a per-layer dump on both arms (the forward already has `NPU_INFER_DUMP_LAYERS`) to find the first layer where they separate, instead of another whole-model guess. Kept as a debug hook rather than removed: it is the check that a future change to the arm is still a GEMM, and it is two lines of print behind an env var. --- npu/forward/include/q4nx_forward.h | 1 + npu/forward/src/q4nx_forward.cpp | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/npu/forward/include/q4nx_forward.h b/npu/forward/include/q4nx_forward.h index 590338f80..fd209e418 100644 --- a/npu/forward/include/q4nx_forward.h +++ b/npu/forward/include/q4nx_forward.h @@ -244,6 +244,7 @@ class Q4nxNpuForward { // Keyed by (wkey, K, N): a wkey alone is not enough -- a second tensor with the same key and a // different shape would reuse the first one's codes. std::map, HostI8Tensor> host_i8_cache_; + int dbg_i8_ab_ = 0; // NPU_DBG_I8_AB: one-GEMM A/B against the f32 arm void host_gemm_i8(const float* A, int a_k, int M, int K, int N, const float* Bmat, int wkey, float* out); bool load_layer(int l); // `a_k` is the activation's REAL width per row (the design's real_k, i.e. the model's hidden or diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 0d1818552..34b5eca74 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -3062,6 +3062,12 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N, it = host_i8_cache_.emplace(key, std::move(t)).first; } const HostI8Tensor& t = it->second; + // One-GEMM A/B (engine#1830): with NPU_DBG_I8_AB set, compare this arm's output against the f32 GEMM for the + // SAME activation and the SAME tensor. cos ~0.99 means the layout and scales are right and a remaining + // degeneracy is an integration issue; cos ~0 means the operands are still wrong. Host only, first few calls. + const bool ab = getenv("NPU_DBG_I8_AB") != nullptr && dbg_i8_ab_ < 4; + std::vector ab_ref; + if (ab) { ab_ref.assign((size_t)M * (size_t)N, 0.0f); q4nx_host_gemm_f32_padded(A, a_k, M, K, N, Bmat, ab_ref.data()); } std::vector a8((size_t)K); for (int m = 0; m < M; m++) { const float* a = A + (size_t)m * (size_t)a_k; @@ -3086,6 +3092,18 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N, dst[(size_t)n] = t.d[(size_t)n] * (ascale * (float)acc); } } + if (ab) { + double dot = 0, na = 0, nb = 0, md = 0; + for (size_t i = 0; i < ab_ref.size(); i++) { + const double a = ab_ref[i], b = out[i]; + dot += a * b; na += a * a; nb += b * b; + md = std::max(md, std::fabs(a - b)); + } + const double cos = dot / (std::sqrt(na * nb) + 1e-30); + std::fprintf(stderr, "[i8ab] wkey=%d K=%d N=%d cos(i8,f32)=%.6f max|d|=%.6g ref[0]=%g i8[0]=%g\n", + wkey, K, N, cos, md, (double)ab_ref.empty() ? 0.0 : (double)ab_ref[0], (double)out[0]); + dbg_i8_ab_++; + } } void Q4nxNpuForward::npu_gemm(int role, const float* A, int K, int N, const float* Bmat, From 8b29068d0cc2f9cb6bfd996063fc3f08d97e5604 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 04:22:30 -0400 Subject: [PATCH 12/14] npu: the integer arm's code cache collided across layers, which is what exploded the residual MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found with the per-GEMM A/B turned into a full scan (`NPU_DBG_I8_AB`, 400 calls): layer 0's four GEMMs came out at cos 0.9993-0.99999 and the **386 calls after them at cos ~0 or negative**, which is why the residual stream grew to 3300x the f32 reference by layer 29 and the model emitted '▁/******/…'. Cause: the codes cache was keyed by `wkey` -- and `wkey` is a weight-cache ROLE key (0, 1, 2, 4 = QKV, O, GU, D) **shared by every layer**, so layers 1-31 reused layer 0's weight image. Keying by the weight POINTER does not fix it either (tried: the host path dequantises each layer into the same per-role scratch, so the pointer repeats). The key is `(layer, wkey, K, N)`. Measured after: * per-GEMM A/B: the worst call over a 3-token request is **cos 0.986** (was 390/390 flagged at cos ~0 or negative); * the served answer becomes **'Paris,'** where the arm previously emitted '▁/******/', and the f32 arm's answer starts with the same token. Still not a gated reference -- the arm quantises from the dequantised matrix rather than the container's own codes (#1808) -- but it now computes the same model to int8 precision instead of another model's weights. --- npu/forward/include/q4nx_forward.h | 2 +- npu/forward/src/q4nx_forward.cpp | 31 ++++++++++++++++++++++++------ 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/npu/forward/include/q4nx_forward.h b/npu/forward/include/q4nx_forward.h index fd209e418..72cc099c7 100644 --- a/npu/forward/include/q4nx_forward.h +++ b/npu/forward/include/q4nx_forward.h @@ -243,7 +243,7 @@ class Q4nxNpuForward { struct HostI8Tensor { int groups = 0; int N = 0; std::vector codes; std::vector d; }; // Keyed by (wkey, K, N): a wkey alone is not enough -- a second tensor with the same key and a // different shape would reuse the first one's codes. - std::map, HostI8Tensor> host_i8_cache_; + std::map, HostI8Tensor> host_i8_cache_; // (layer, wkey, K, N) int dbg_i8_ab_ = 0; // NPU_DBG_I8_AB: one-GEMM A/B against the f32 arm void host_gemm_i8(const float* A, int a_k, int M, int K, int N, const float* Bmat, int wkey, float* out); bool load_layer(int l); diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 34b5eca74..19dc7478a 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -3050,7 +3050,17 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N, // different model: it answered "ikt" to "The capital of France is" and then locked onto one token; // * the image and its per-column scales are cached per (wkey, K, N), because quantising 7B per call cost // 7.5 G scalar operations and the arm never finished a token in a 9-minute ticket. - const auto key = std::make_tuple(wkey, K, N); + // Keyed by the WEIGHT POINTER, not by `wkey`: `wkey` is a weight-cache role key (0, 1, 2, 4 -- QKV, O, + // GU, D), so every layer shares it, and keying by it made layer 1 onward reuse layer 0's codes. Measured + // 2026-11-10 with the per-GEMM A/B: layers 0's four GEMMs at cos 0.9993-0.99999 and **390 further calls + // at cos ~0 or negative**, with the residual exploding to 3300x the reference by layer 29. The pointer + // identifies the tensor in host mode, where the model owns its weights for the process's life. + // Keyed by (layer, role/weight key, K, N). Neither `wkey` alone nor the weight POINTER works: `wkey` is a + // weight-cache ROLE key shared by every layer (0, 1, 2, 4 -- QKV, O, GU, D), and in host mode the caller + // dequantises each layer into the same per-role scratch, so the pointer repeats too. Measured 2026-11-10 with + // the per-GEMM A/B: with either of those keys, layer 0's four GEMMs came out at cos 0.9993-0.99999 and the + // 386 calls after them at cos ~0 or negative, with the residual stream exploding to 3300x the reference. + const auto key = std::make_tuple(cur_layer_, wkey, K, N); auto it = host_i8_cache_.find(key); if (it == host_i8_cache_.end()) { HostI8Tensor t; @@ -3062,24 +3072,31 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N, it = host_i8_cache_.emplace(key, std::move(t)).first; } const HostI8Tensor& t = it->second; + std::vector padded_a; // One-GEMM A/B (engine#1830): with NPU_DBG_I8_AB set, compare this arm's output against the f32 GEMM for the // SAME activation and the SAME tensor. cos ~0.99 means the layout and scales are right and a remaining // degeneracy is an integration issue; cos ~0 means the operands are still wrong. Host only, first few calls. - const bool ab = getenv("NPU_DBG_I8_AB") != nullptr && dbg_i8_ab_ < 4; + const bool ab = getenv("NPU_DBG_I8_AB") != nullptr && dbg_i8_ab_ < 400; std::vector ab_ref; if (ab) { ab_ref.assign((size_t)M * (size_t)N, 0.0f); q4nx_host_gemm_f32_padded(A, a_k, M, K, N, Bmat, ab_ref.data()); } std::vector a8((size_t)K); for (int m = 0; m < M; m++) { + // The caller's row is `a_k` wide and laid out at stride `a_k`; the GEMM contracts over K. That is the + // whole reason q4nx_host_gemm_f32_padded() exists (engine#845): reading a[k] for k < K straight out of an + // a_k-wide buffer runs off the row into the next one. So pad onto a K-wide row first, exactly as the f32 + // arm does, and quantise from that. const float* a = A + (size_t)m * (size_t)a_k; + padded_a.assign((size_t)K, 0.0f); + for (int k = 0; k < (a_k < K ? a_k : K); k++) padded_a[(size_t)k] = a[k]; float amax = 0.0f; for (int k = 0; k < K; k++) { - const float v = std::fabs(a[k]); + const float v = std::fabs(padded_a[(size_t)k]); if (std::isfinite(v) && v > amax) amax = v; } const float ascale = (amax >= 1e-12f) ? amax / 127.0f : 1.0f; const float tis = 1.0f / ascale; for (int k = 0; k < K; k++) { - float v = a[k]; + float v = padded_a[(size_t)k]; if (!std::isfinite(v)) v = 0.0f; int q = (int)std::round(v * tis); a8[(size_t)k] = (int8_t)(q > 127 ? 127 : (q < -127 ? -127 : q)); @@ -3100,8 +3117,10 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N, md = std::max(md, std::fabs(a - b)); } const double cos = dot / (std::sqrt(na * nb) + 1e-30); - std::fprintf(stderr, "[i8ab] wkey=%d K=%d N=%d cos(i8,f32)=%.6f max|d|=%.6g ref[0]=%g i8[0]=%g\n", - wkey, K, N, cos, md, (double)ab_ref.empty() ? 0.0 : (double)ab_ref[0], (double)out[0]); + if (cos < 0.999 || dbg_i8_ab_ < 4) + std::fprintf(stderr, "[i8ab] call=%d wkey=%d K=%d N=%d cos(i8,f32)=%.6f max|d|=%.6g ref[0]=%g i8[0]=%g\n", + dbg_i8_ab_, wkey, K, N, cos, md, + ab_ref.empty() ? 0.0 : (double)ab_ref[0], (double)out[0]); dbg_i8_ab_++; } } From 10fc175efa8afea06b388dfe611ddf09f5bd45cf Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 05:13:25 -0400 Subject: [PATCH 13/14] npu: float ordering cannot explain the integer arm's residue to the device (measured: the gap is 0) The arm is 0.30 max|d| from the device on a 4096-family gate while being 0.9993+ in cosine. The cheapest explanation I had left was that the device multiplies its epilogue as ((cm * asc) * gs) and this arm as (gs * (asc * cm)) -- float multiplication is not associative, so the two orders can differ. Measured instead of assumed: the A/B now computes both orders for every element of every call and reports the largest gap. Over a whole request it is **exactly 0** on all four roles. So ordering is excluded, and the remaining difference must be in the OPERANDS -- a code or a scale that differs between the device's image and q4nx_pack_B_int8_blocked()'s, despite both starting from the same f32 matrix. That narrows engine#1808 to one measurement: dump the device's post-readback int32 (`kctx.Cm`) for one GEMM and diff it against the arm's accumulator, as integers. Kept the order probe behind the same env var (NPU_DBG_I8_AB) so the exclusion is repeatable. --- npu/forward/src/q4nx_forward.cpp | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 19dc7478a..dd7a90a4b 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -3073,6 +3073,7 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N, } const HostI8Tensor& t = it->second; std::vector padded_a; + float order_gap_ = 0.0f; // One-GEMM A/B (engine#1830): with NPU_DBG_I8_AB set, compare this arm's output against the f32 GEMM for the // SAME activation and the SAME tensor. cos ~0.99 means the layout and scales are right and a remaining // degeneracy is an integration issue; cos ~0 means the operands are still wrong. Host only, first few calls. @@ -3107,6 +3108,12 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N, for (int k = 0; k < K; k++) acc += (int32_t)((int)t.codes[q4nx_layout::blocked_index((size_t)k, (size_t)n, K, N)] * (int)a8[(size_t)k]); dst[(size_t)n] = t.d[(size_t)n] * (ascale * (float)acc); + // How much can float ORDER alone move this value? The device multiplies ((cm * asc) * gs); this arm + // multiplies (gs * (asc * cm)). Track the gap so "the residue is just multiplication order" can be + // checked instead of asserted. + const float alt = ((float)acc * ascale) * t.d[(size_t)n]; + const float gap = std::fabs(alt - dst[(size_t)n]); + if (gap > order_gap_) order_gap_ = gap; } } if (ab) { @@ -3118,9 +3125,11 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N, } const double cos = dot / (std::sqrt(na * nb) + 1e-30); if (cos < 0.999 || dbg_i8_ab_ < 4) - std::fprintf(stderr, "[i8ab] call=%d wkey=%d K=%d N=%d cos(i8,f32)=%.6f max|d|=%.6g ref[0]=%g i8[0]=%g\n", - dbg_i8_ab_, wkey, K, N, cos, md, + std::fprintf(stderr, "[i8ab] call=%d wkey=%d K=%d N=%d cos(i8,f32)=%.6f max|d|=%.6g " + "epilogue_order_gap=%.3g ref[0]=%g i8[0]=%g\n", + dbg_i8_ab_, wkey, K, N, cos, md, (double)order_gap_, ab_ref.empty() ? 0.0 : (double)ab_ref[0], (double)out[0]); + order_gap_ = 0.0f; dbg_i8_ab_++; } } From 75299d963fb198d896502abd0ce6afd7718c75bb Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 05:38:13 -0400 Subject: [PATCH 14/14] npu: the device's int32 accumulators and the arm's are bit-identical for a whole token's pass Added env-gated int32 dumps on both sides (`NPU_I8_INT32_DUMP`: the device's post-readback `kctx.Cm` row, the arm's accumulator) and compared them as integers over a one-token request on the Mistral-7B 1.9 container: * **the first 400 calls are exactly equal** -- zero difference on every element. That is a whole pass over the model (12 calls per layer x 32 layers plus the head), i.e. the GEMMs' integer arithmetic matches the device bit for bit; * **from call 400 (the second token's first QKV, N=6144) every call differs**, with argmax no longer matching. So the operands are identical while the model is on the same state, and the divergence begins when the second token reuses the KV cache. That points the remaining question at STATE (what the second token's activation is built from), not at the weight codes, the packing, the activation scale or the epilogue -- all four of which are now excluded by measurement rather than argument. This is why the earlier "cos 0.9993 device vs int8" reading is consistent: the first step agrees, and every step after it is a consequence of the first state difference. Dumps are off unless the variable is set; nothing runs in the shipped path. --- npu/forward/src/q4nx_forward.cpp | 13 +++++++++++++ npu/target/xdna/xdna_target.cpp | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index dd7a90a4b..76b5f51ca 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -3074,6 +3074,8 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N, const HostI8Tensor& t = it->second; std::vector padded_a; float order_gap_ = 0.0f; + static int i8_dump_call_ = 0; + std::vector acc_row; // One-GEMM A/B (engine#1830): with NPU_DBG_I8_AB set, compare this arm's output against the f32 GEMM for the // SAME activation and the SAME tensor. cos ~0.99 means the layout and scales are right and a remaining // degeneracy is an integration issue; cos ~0 means the operands are still wrong. Host only, first few calls. @@ -3103,11 +3105,13 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N, a8[(size_t)k] = (int8_t)(q > 127 ? 127 : (q < -127 ? -127 : q)); } float* dst = out + (size_t)m * (size_t)N; + acc_row.assign((size_t)N, 0); for (int n = 0; n < N; n++) { int32_t acc = 0; for (int k = 0; k < K; k++) acc += (int32_t)((int)t.codes[q4nx_layout::blocked_index((size_t)k, (size_t)n, K, N)] * (int)a8[(size_t)k]); dst[(size_t)n] = t.d[(size_t)n] * (ascale * (float)acc); + acc_row[(size_t)n] = acc; // How much can float ORDER alone move this value? The device multiplies ((cm * asc) * gs); this arm // multiplies (gs * (asc * cm)). Track the gap so "the residue is just multiplication order" can be // checked instead of asserted. @@ -3115,7 +3119,16 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N, const float gap = std::fabs(alt - dst[(size_t)n]); if (gap > order_gap_) order_gap_ = gap; } + if (const char* idir = getenv("NPU_I8_INT32_DUMP")) { + char path[4096]; + std::snprintf(path, sizeof path, "%s/i8_%03d_m%d.bin", idir, i8_dump_call_, m); + if (std::FILE* f = std::fopen(path, "wb")) { + std::fwrite(acc_row.data(), sizeof(int32_t), (size_t)N, f); + std::fclose(f); + } + } } + if (getenv("NPU_I8_INT32_DUMP")) i8_dump_call_++; if (ab) { double dot = 0, na = 0, nb = 0, md = 0; for (size_t i = 0; i < ab_ref.size(); i++) { diff --git a/npu/target/xdna/xdna_target.cpp b/npu/target/xdna/xdna_target.cpp index fe4c4ecae..79eb54963 100644 --- a/npu/target/xdna/xdna_target.cpp +++ b/npu/target/xdna/xdna_target.cpp @@ -1153,6 +1153,19 @@ void XdnaTarget::npu_gemm_rows(const GemvArgs& ar, std::vector& out, std: const int32_t* cm = kctx.Cm + (size_t)m * kctx.ND; const float asc = ascales[(size_t)m]; float* o = out.data() + (size_t)m * N; + // engine#1808: the last measurement between "the device is right to int8 precision" and "bit-exact at the + // accumulator" is the INTEGER comparison -- dump the post-readback int32 rows so they can be diffed against + // a host arm's accumulators directly. Env-gated, off by default, one file per call. + if (const char* idir = getenv("NPU_I8_INT32_DUMP")) { + static int call = 0; + char path[4096]; + std::snprintf(path, sizeof path, "%s/dev_%03d_m%d.bin", idir, call, m); + if (std::FILE* f = std::fopen(path, "wb")) { + std::fwrite(cm, sizeof(int32_t), (size_t)N, f); + std::fclose(f); + } + if (m + 1 == M) call++; + } for (int n = 0; n < N; n++) o[n] = (float)cm[n] * asc * (*gs)[(size_t)n]; }