Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -847,6 +847,10 @@ add_test(NAME route_port_lock COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/
# the whole run loop against canned completions, and re-derives the summary from the written transcript. Stdlib
# only, with no route, no socket, no dataset fetch and no device.
add_test(NAME gsm8k_eval COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/gsm8k_eval_test.py)
# engine#1808's integer host arm is pinned inside tests/npu_pack_test.cpp (ctest npu_pack): that binary is
# already built by ci.yml, and a new ctest target would need a `.github/workflows/ci.yml` edit, which this
# lane's token cannot push (the OAuth app has no `workflow` scope) -- the ci_targets check caught exactly
# that, with both failures ("ci_targets" and "npu_host_gemm_i8 Not Run").
# The repack's tile-order claim (engine#1110): `npu_plan.py` writes container.tile_order=engine-1 for every
# plan, so the load-time guard (#1130) is green on a container whose TILES are another order. This pins the
# mapping from q4nx_diff's verdict to (stamp value, fatal), stdlib only: no model, no numpy, no device.
Expand Down Expand Up @@ -1095,13 +1099,18 @@ if(ONEBIT_OGA_PACK)
add_test(NAME npu_pack_import_oga
COMMAND npu_pack_import_test ${CMAKE_SOURCE_DIR}/tests/golden ${ONEBIT_OGA_PACK})
endif()
add_executable(npu_pack_test tests/npu_pack_test.cpp)
# q4nx_pack.c is listed here rather than reached through onebit_npu: the integer host arm's checks
# (engine#1808) call q4nx_host_gemm_i8*() directly, and those live in that file.
add_executable(npu_pack_test tests/npu_pack_test.cpp npu/forward/src/q4nx_pack.c npu/forward/src/q4nx_dequant.c)
add_executable(flm_head_import tools/flm_head_import.cpp)
target_link_libraries(flm_head_import PRIVATE onebit_npu)
add_executable(npu_flm_head_import_test tests/npu_flm_head_import_test.cpp)
target_link_libraries(npu_flm_head_import_test PRIVATE onebit_npu)
add_test(NAME npu_flm_head_import COMMAND npu_flm_head_import_test ${CMAKE_SOURCE_DIR}/tests/golden/flm-i8)
target_link_libraries(npu_pack_test PRIVATE onebit_npu)
# engine#1808: the integer host arm's checks live in this binary (see above), so it needs the header and the
# golden fixture that pins them.
target_include_directories(npu_pack_test PRIVATE npu/forward/include ${CMAKE_SOURCE_DIR}/tests)

# The device weight layout contract (q4nx_weight_layout): pure index math, no device
# and no runtime. CI builds an explicit target list, so the check compiles itself with
Expand Down
13 changes: 13 additions & 0 deletions npu/forward/include/q4nx_forward.h
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
#include <memory>
#include <string>
#include <vector>
#include <tuple>

#include "moe_container.h" // the two MoE container conventions (engine#1035)
#include "model.h"
Expand Down Expand Up @@ -233,6 +234,18 @@ class Q4nxNpuForward {
bool init_impl(xrt::device* dev, const char* model_path,
const std::string& artifact_dir);
bool host_gemm_ = false;
// NPU_INFER_HOST_GEMM=i8 (engine#1808): the host arm accumulates int8 x int4 in EXACT int32, which is
// the only arm a kernel family can be bit-exact against -- f32 cannot match an int8 device by
// construction. It cannot see a wrong weight layout (the f32 arm covers that); it can see a wrong
// quantisation, a wrong 128-K group boundary, an int8 intermediate or a saturated accumulator.
bool host_gemm_i8_ = false;
// Weight codes + per-128-K multipliers for the integer host arm, quantised once per tensor (engine#1809).
struct HostI8Tensor { int groups = 0; int N = 0; std::vector<int8_t> codes; std::vector<float> d; };
// Keyed by (wkey, K, N): a wkey alone is not enough -- a second tensor with the same key and a
// different shape would reuse the first one's codes.
std::map<std::tuple<int, int, int, int>, HostI8Tensor> host_i8_cache_; // (layer, wkey, K, N)
int dbg_i8_ab_ = 0; // NPU_DBG_I8_AB: one-GEMM A/B against the f32 arm
void host_gemm_i8(const float* A, int a_k, int M, int K, int N, const float* Bmat, int wkey, float* out);
bool load_layer(int l);
// `a_k` is the activation's REAL width per row (the design's real_k, i.e. the model's hidden or
// intermediate size). It is K whenever the model is already on the container's 256-column K
Expand Down
28 changes: 28 additions & 0 deletions npu/forward/include/q4nx_pack.h
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,34 @@ void q4nx_pad_a_f32(const float* A, int M, int real_k, int K, float* out);
void q4nx_host_gemm_f32_padded(const float* A, int a_k, int M, int K, int N, const float* Bmat,
float* out);

/* The integer arm of the same host path (engine#1808): int8 activation x int4 weight codes -> exact int32,
* which is what the kernel computes. Written for the bit-exactness gate, because the f32 host path cannot
* be bit-exact with an int8 device arm by construction; scales are applied by the caller on both sides so
* a scale mistake cannot hide inside the integer comparison. `codes` is [K][N], one int8 per element
* holding an int4 value in [-8, 7], exactly as the importer's packB lays it down. */
void q4nx_host_gemm_i8(const int8_t* a8, int K, const int8_t* codes, int N, int32_t* out32);

/* The whole GEMM in int8 x int4 for the gate's `i8` host arm, quantising both sides here with the engine's
* own rule. It cannot see a wrong weight LAYOUT (the f32 arm covers that); what it can see is a wrong int4
* quantisation, a wrong 128-K group boundary, an int8 intermediate or a saturated accumulator -- i.e. the
* kernel's arithmetic, exactly. int32 out, M*N, no scaling applied. */
void q4nx_host_gemm_i8_from_f32(const float* A, int a_k, int M, int K, int N, const float* Bmat,
int32_t* out32);

/* The reusable form: `codes` and `d` are quantised once per tensor by the caller (a per-call quantisation of
* the weights costs ~7.5 G scalar operations per 7B token, measured — it never finished a token in a
* 9-minute ticket), and this applies the kernel's epilogue: ascale * sum_g d[g][n] * sum_{k in g} code*a8,
* with the inner sum exact int32 and no float inside it. */
void q4nx_host_gemm_i8_grouped(const int8_t* a8, int K, const int8_t* codes, const float* d,
int groups, int N, float ascale, float* out);

/* The form that matches `packB_blocked()`: one multiplier per weight COLUMN (amax over the whole K, codes int8
* in [-127,127]), inner sum exact int32. This is the arm a bit-exactness gate can use, because the operands are
* the ones our own packing produces; the per-128-K int4 form above re-quantises into a different model and is
* kept only for the arithmetic tests. */
void q4nx_host_gemm_i8_col(const int8_t* a8, int K, const int8_t* codes, const float* d, int N, float ascale,
float* out);


/* Architecture dims a design set is derived from. */
typedef struct {
Expand Down
132 changes: 131 additions & 1 deletion npu/forward/src/q4nx_forward.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,9 @@
#include "q4nx_device.h" // an imported q4nx-2 container's device form (engine#1185)
#include "q4nx_semantics.h"
#include "q4nx_pack.h"
#include "q4nx_weight_layout.h"

#include <tuple>

#if defined(_OPENMP)
#include <omp.h>
Expand Down Expand Up @@ -424,7 +427,14 @@

bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path,
const std::string& dir) {
if (getenv("NPU_INFER_HOST_GEMM")) host_gemm_ = true;
if (const char* hg = getenv("NPU_INFER_HOST_GEMM")) {
host_gemm_ = true;
// engine#1808: "i8" selects the integer arm, whose int32 accumulation is the only thing a
// kernel family can be bit-exact against (an int8 device cannot match an f32 host by
// construction). Any other value keeps the f32 path, which is what every existing caller and
// every row measured so far used.
host_gemm_i8_ = (hg[0] == 'i' && hg[1] == '8');
}
// The launch layer. A1: everything below that touches a device talks to this, and the settings the
// container declares (weight layout, pb16i image dir, Hadamard fold) are pushed into it as they are
// parsed. A2 turns this into NpuTarget::open(plan, kernels_dir).
Expand Down Expand Up @@ -3025,6 +3035,118 @@
// packed-weight BO cache, pb16i images, Hadamard fold. What stays here is the model's own fallback, the
// plain float GEMM of init_host() -- that is a property of the model, not of any NPU.
// --------------------------------------------------------------------------
// The integer host arm (engine#1808/#1809). Two decisions here, both measured:
// * the weight codes and their per-128-K multipliers are quantised ONCE per tensor (`wkey`) and cached --
// doing it per call made a 7B token ~7.5 G scalar operations and the arm never finished a token inside a
// 9-minute ticket;
// * the inner sum is exact int32 with no float in it (q4nx_host_gemm_i8_grouped), so the comparison against
// the device is about accumulation and epilogue, not about a float both sides happen to share.
void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N, const float* Bmat,
int wkey, float* out) {
// engine#1808/#1830: this arm exists so a kernel family can be gated "bit-exact against the host
// reference", which f32 cannot be. Two decisions, both measured:
// * the weight image comes from THE ENGINE'S OWN PACKER (`q4nx_pack_B_int8_blocked`, the same code the
// device path uses), not from a re-quantisation of my own -- the first version did that and produced a
// different model: it answered "ikt" to "The capital of France is" and then locked onto one token;
// * the image and its per-column scales are cached per (wkey, K, N), because quantising 7B per call cost
// 7.5 G scalar operations and the arm never finished a token in a 9-minute ticket.
// Keyed by the WEIGHT POINTER, not by `wkey`: `wkey` is a weight-cache role key (0, 1, 2, 4 -- QKV, O,
// GU, D), so every layer shares it, and keying by it made layer 1 onward reuse layer 0's codes. Measured
// 2026-11-10 with the per-GEMM A/B: layers 0's four GEMMs at cos 0.9993-0.99999 and **390 further calls
// at cos ~0 or negative**, with the residual exploding to 3300x the reference by layer 29. The pointer
// identifies the tensor in host mode, where the model owns its weights for the process's life.
// Keyed by (layer, role/weight key, K, N). Neither `wkey` alone nor the weight POINTER works: `wkey` is a
// weight-cache ROLE key shared by every layer (0, 1, 2, 4 -- QKV, O, GU, D), and in host mode the caller
// dequantises each layer into the same per-role scratch, so the pointer repeats too. Measured 2026-11-10 with
// the per-GEMM A/B: with either of those keys, layer 0's four GEMMs came out at cos 0.9993-0.99999 and the
// 386 calls after them at cos ~0 or negative, with the residual stream exploding to 3300x the reference.
const auto key = std::make_tuple(cur_layer_, wkey, K, N);
auto it = host_i8_cache_.find(key);
if (it == host_i8_cache_.end()) {
HostI8Tensor t;
t.N = N;
t.codes.assign((size_t)K * (size_t)N, 0);
t.d.assign((size_t)N, 0.0f);
double ssum = 0.0;
q4nx_pack_B_int8_blocked(t.codes.data(), Bmat, K, N, K, N, /*blocked=*/1, t.d.data(), &ssum);
it = host_i8_cache_.emplace(key, std::move(t)).first;
}
const HostI8Tensor& t = it->second;
std::vector<float> padded_a;
float order_gap_ = 0.0f;
static int i8_dump_call_ = 0;
std::vector<int32_t> acc_row;
// One-GEMM A/B (engine#1830): with NPU_DBG_I8_AB set, compare this arm's output against the f32 GEMM for the
// SAME activation and the SAME tensor. cos ~0.99 means the layout and scales are right and a remaining
// degeneracy is an integration issue; cos ~0 means the operands are still wrong. Host only, first few calls.
const bool ab = getenv("NPU_DBG_I8_AB") != nullptr && dbg_i8_ab_ < 400;
std::vector<float> ab_ref;
if (ab) { ab_ref.assign((size_t)M * (size_t)N, 0.0f); q4nx_host_gemm_f32_padded(A, a_k, M, K, N, Bmat, ab_ref.data()); }
std::vector<int8_t> a8((size_t)K);
for (int m = 0; m < M; m++) {
// The caller's row is `a_k` wide and laid out at stride `a_k`; the GEMM contracts over K. That is the
// whole reason q4nx_host_gemm_f32_padded() exists (engine#845): reading a[k] for k < K straight out of an
// a_k-wide buffer runs off the row into the next one. So pad onto a K-wide row first, exactly as the f32
// arm does, and quantise from that.
const float* a = A + (size_t)m * (size_t)a_k;
padded_a.assign((size_t)K, 0.0f);
for (int k = 0; k < (a_k < K ? a_k : K); k++) padded_a[(size_t)k] = a[k];
float amax = 0.0f;
for (int k = 0; k < K; k++) {
const float v = std::fabs(padded_a[(size_t)k]);
if (std::isfinite(v) && v > amax) amax = v;
}
const float ascale = (amax >= 1e-12f) ? amax / 127.0f : 1.0f;
const float tis = 1.0f / ascale;
for (int k = 0; k < K; k++) {
float v = padded_a[(size_t)k];
if (!std::isfinite(v)) v = 0.0f;
int q = (int)std::round(v * tis);
a8[(size_t)k] = (int8_t)(q > 127 ? 127 : (q < -127 ? -127 : q));
}
float* dst = out + (size_t)m * (size_t)N;
acc_row.assign((size_t)N, 0);
for (int n = 0; n < N; n++) {
int32_t acc = 0;
for (int k = 0; k < K; k++)
acc += (int32_t)((int)t.codes[q4nx_layout::blocked_index((size_t)k, (size_t)n, K, N)] * (int)a8[(size_t)k]);
dst[(size_t)n] = t.d[(size_t)n] * (ascale * (float)acc);
acc_row[(size_t)n] = acc;
// How much can float ORDER alone move this value? The device multiplies ((cm * asc) * gs); this arm
// multiplies (gs * (asc * cm)). Track the gap so "the residue is just multiplication order" can be
// checked instead of asserted.
const float alt = ((float)acc * ascale) * t.d[(size_t)n];
const float gap = std::fabs(alt - dst[(size_t)n]);
if (gap > order_gap_) order_gap_ = gap;
}
if (const char* idir = getenv("NPU_I8_INT32_DUMP")) {
char path[4096];
std::snprintf(path, sizeof path, "%s/i8_%03d_m%d.bin", idir, i8_dump_call_, m);
if (std::FILE* f = std::fopen(path, "wb")) {
std::fwrite(acc_row.data(), sizeof(int32_t), (size_t)N, f);
std::fclose(f);
}
}
}
if (getenv("NPU_I8_INT32_DUMP")) i8_dump_call_++;
if (ab) {
double dot = 0, na = 0, nb = 0, md = 0;
for (size_t i = 0; i < ab_ref.size(); i++) {
const double a = ab_ref[i], b = out[i];
dot += a * b; na += a * a; nb += b * b;
md = std::max(md, std::fabs(a - b));
}
const double cos = dot / (std::sqrt(na * nb) + 1e-30);
if (cos < 0.999 || dbg_i8_ab_ < 4)
std::fprintf(stderr, "[i8ab] call=%d wkey=%d K=%d N=%d cos(i8,f32)=%.6f max|d|=%.6g "
"epilogue_order_gap=%.3g ref[0]=%g i8[0]=%g\n",
dbg_i8_ab_, wkey, K, N, cos, md, (double)order_gap_,
ab_ref.empty() ? 0.0 : (double)ab_ref[0], (double)out[0]);
order_gap_ = 0.0f;
dbg_i8_ab_++;
}
}

void Q4nxNpuForward::npu_gemm(int role, const float* A, int K, int N, const float* Bmat,
std::vector<float>& out, int wkey, const char* sc, int a_k) {
if (host_gemm_) {
Expand All @@ -3033,6 +3155,10 @@
// unpadded A. (In review of this PR the split moved the pad into the target and left this
// branch reading past the caller's buffer at a padded design.)
out.assign((size_t)N, 0.0f);
if (host_gemm_i8_) {
host_gemm_i8(A, a_k, 1, K, N, Bmat, wkey, out.data());
return;
}
q4nx_host_gemm_f32_padded(A, a_k, 1, K, N, Bmat, out.data());
return;
}
Expand All @@ -3052,6 +3178,10 @@
// Same pad, and here it is also a stride: the rows form used to index A at `m * K` while the
// caller's buffer is laid out at `m * a_k`, so every row past the first read the wrong row.
out.assign((size_t)M * N, 0.0f);
if (host_gemm_i8_) {
host_gemm_i8(A, a_k, M, K, N, Bmat, wkey, out.data());
return;
}
q4nx_host_gemm_f32_padded(A, a_k, M, K, N, Bmat, out.data());
return;
}
Expand Down
Loading
Loading