diff --git a/CMakeLists.txt b/CMakeLists.txt index 16cbce9bc..00b96241f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -692,7 +692,7 @@ option(ONEBIT_NPU_HOST "Build the NPU host library and its tests" ${_onebit_npu_ if(ONEBIT_NPU_HOST) find_package(PkgConfig REQUIRED) pkg_check_modules(PCRE2 REQUIRED IMPORTED_TARGET libpcre2-8) -add_library(onebit_npu STATIC npu/file_map.cpp npu/flm_head_import.cpp npu/full_elf.cpp npu/model.cpp npu/vision_pack.cpp +add_library(onebit_npu STATIC npu/file_map.cpp npu/flm_head_import.cpp npu/full_elf.cpp npu/model.cpp npu/vision_pack.cpp npu/vision_tower.cpp npu/moe.cpp npu/pack.cpp npu/pack_import.cpp npu/private_route.cpp npu/q4nx.cpp npu/q4nx_device.cpp npu/tokenizer.cpp) target_include_directories(onebit_npu PUBLIC npu) @@ -1213,6 +1213,11 @@ add_executable(npu_vision_pack_test tests/npu_vision_pack_test.cpp) target_link_libraries(npu_vision_pack_test PRIVATE onebit_npu) add_test(NAME npu_vision_pack COMMAND npu_vision_pack_test) add_dependencies(onebit npu_vision_pack_test) +# The host float32 tower against goldens from the transformers implementation (see the test's header). +add_executable(npu_vision_tower_test tests/npu_vision_tower_test.cpp) +target_link_libraries(npu_vision_tower_test PRIVATE onebit_npu) +add_test(NAME npu_vision_tower COMMAND npu_vision_tower_test) +add_dependencies(onebit npu_vision_tower_test) add_executable(npu_q4nx_test tests/npu_q4nx_test.cpp) target_include_directories(npu_q4nx_test PRIVATE app) target_link_libraries(npu_q4nx_test PRIVATE onebit_npu) diff --git a/docs/npu.md b/docs/npu.md index 76f584b16..1c4396014 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1462,7 +1462,7 @@ tensors 347, all BF16, 1135109120 B of payload, 0 B unaccounted patch embed 14 x 14 pixels x 3 channels -> 1152 positions 4900 (a 70 x 70 grid; image 980) blocks 27 at width 1152, 16 heads x 72, MLP 4304 (4608 stored) -vit_merger attention at 1152, MLP 4608 -> 17216 (17280 stored) -> 1152 +vit_merger after block 6; attention at 1152, MLP 4608 -> 17216 (17280 stored) -> 1152 merger 4608 -> 4608 -> 1024 (the text model's hidden size) source_file mmproj-model-f16.gguf source_sha256 fb9cc047a3cdad5317fd8433a4aeda2d22f00f02ca4bd93b34ad84165b25a900 @@ -1470,8 +1470,7 @@ note config.json states MINICPM_V_VISION_MLP_INTERMEDIATE_SIZE 4304 note config.json states MINICPM_V_VIT_MERGER_INTERMEDIATE_SIZE 17216; model.visual.vit_merger.mlp.linear_fc1.weight has 17280 rows, the last 64 all zero: the container pads the model's width, and sizes come from the tensor ``` -This is a reader, not a forward: nothing runs the tower yet. What it settles is what a forward may -rely on. +What the reader settles is what a forward may rely on. - **Sizes come from the tensors.** The geometry is derived from the tensors' own shapes, and every tensor must have the shape the others imply. @@ -1508,6 +1507,41 @@ Pinned by `tests/npu_vision_pack_test.cpp` on crafted containers; `ONEBIT_MINICP the published pack's own numbers. The source comparison above was run by hand and is not a test in the tree. +**The tower on the host (`npu/vision_tower.h`).** `vision::tower_forward(pack, pixels, gh, gw)` runs the +tower in float32 on the CPU: normalised pixels for a `gh × gw` patch grid in, `(gh/4) × (gw/4)` rows at +the text model's width out. It is a reference for whatever runs the tower on the NPU later, not a +serving path; nothing calls it from a serve, and it does no image decoding, resizing or slicing. The +order of operations is the transformers implementation's (`transformers.models.minicpmv4_7`): + +1. patch embedding (a Conv2d whose kernel is its stride), patches row-major; +2. plus one learned position row per patch, picked **nearest** on the 70 × 70 table: row + `floor(y·70/gh)`, column `floor(x·70/gw)`. No interpolation and no rotary term; +3. blocks 0–6: LayerNorm (eps 1e-6), full attention, MLP with the tanh GELU; +4. the `vit_merger`, once, after block `insert_layer_id` (6): attention inside each 2 × 2 window only, + then per window an MLP over the four tokens side by side plus their mean. The grid halves; +5. blocks 7–26 on the halved grid, then `merger.norm`; +6. the merger: 2 × 2 groups again, `pre_norm`, `linear_fc1`, the exact (erf) GELU, `linear_fc2`. + +So both grid extents must be multiples of 4, and anything else is refused. The padded MLPs are read at +the tensor's stride and the model's width. + +Measured against that implementation (transformers 5.22.2, torch 2.13 CPU, float32, eager attention) +loaded with each pack's own weights, same input (`pixel(c, y, x) = sin(0.37c + 0.11y + 0.05x + +0.003xy)`), 2026-11-10: + +| pack | grid | tower before the merger: max \|d\|, corr | output: max \|d\|, corr | +|---|---|---|---| +| 1B | 8 × 12 | 2.9e-5, 1.000000000 | 1.1e-6, 1.000000000 | +| 1B | 16 × 20 | 1.1e-5, 1.000000000 | 2.4e-6, 1.000000000 | +| 35B-A3B | 16 × 20 | 9.5e-6, 1.000000000 | 9.5e-7, 1.000000000 | + +The 16 × 20 grid took 5.5 s and 1.25 GB of memory for this code on one thread; a full 70 × 70 slice +has 61 times the tokens and its attention grows with the square, which is why this is a reference. +`tests/npu_vision_tower_test.cpp` carries a small tower with LCG weights and that implementation's +output for it as goldens, so the arithmetic is checked in CI without a pack; with +`ONEBIT_MINICPM_PACK` and `ONEBIT_MINICPM_ORACLE` set it repeats the comparison above. The reference +run itself needs transformers and torch and is not in the tree. + ## Serving The 1bit NPU engine runs the model on the XDNA 2 NPU. With a release package that carries the NPU diff --git a/npu/vision_pack.cpp b/npu/vision_pack.cpp index 4304b5c69..b89f5878c 100644 --- a/npu/vision_pack.cpp +++ b/npu/vision_pack.cpp @@ -299,6 +299,22 @@ void Pack::open(const std::string& container_path, const std::string& config_jso fail("config.json states the text model's hidden_size " + std::to_string(c["hidden_size"].get()) + " but the merger projects to " + std::to_string(geom_.out_hidden) + ": this tower is not this text model's"); + { + // top level first (the model's own config key), then the vision_config spelling; both present must agree + int top = -1, inner = -1; + const bool have_top = c.contains("insert_layer_id") && c["insert_layer_id"].is_number_integer(); + if (have_top) top = int(c["insert_layer_id"].get()); + const bool have_inner = vc.contains("MINICPM_V_INSERT_LAYER_ID") && vc["MINICPM_V_INSERT_LAYER_ID"].is_number_integer(); + if (have_inner) inner = int(vc["MINICPM_V_INSERT_LAYER_ID"].get()); + if (have_top && have_inner && top != inner) + fail("config.json states insert_layer_id " + std::to_string(top) + " and MINICPM_V_INSERT_LAYER_ID " + + std::to_string(inner) + ": one tower, two merge points"); + if (have_top || have_inner) geom_.insert_layer = have_top ? top : inner; + if ((have_top || have_inner) && (geom_.insert_layer < 0 || geom_.insert_layer >= geom_.layers)) + fail("config.json states insert_layer_id " + std::to_string(geom_.insert_layer) + " but the container holds " + + std::to_string(geom_.layers) + " blocks"); + } + // An MLP's stated width against its tensors. `stems` are the MLPs that share the width (every block, or the // vit_merger alone): fc1 is [width, in] with a [width] bias, fc2 is [out, width]. const auto zero_from = [&](const std::string& name, int64_t rows, int64_t cols, int64_t from, bool by_row) { @@ -382,7 +398,7 @@ int print_info(const std::string& pack_dir) { std::printf("positions %d (a %d x %d grid; image %d)\n", g.pos_rows, g.grid, g.grid, g.grid * g.patch); std::printf("blocks %d at width %d, %d heads x %d, MLP %d (%d stored)\n", g.layers, g.hidden, g.heads, g.head_dim, g.model_intermediate, g.intermediate); - std::printf("vit_merger attention at %d, MLP %d -> %d (%d stored) -> %d\n", g.hidden, g.merge_hidden, + std::printf("vit_merger after block %d; attention at %d, MLP %d -> %d (%d stored) -> %d\n", g.insert_layer, g.hidden, g.merge_hidden, g.model_vit_merger_intermediate, g.vit_merger_intermediate, g.hidden); std::printf("merger %d -> %d -> %d (the text model's hidden size)\n", g.merge_hidden, g.merge_hidden, g.out_hidden); for (const char* k : {"source_file", "source_sha256", "source_dtype", "converter"}) diff --git a/npu/vision_pack.h b/npu/vision_pack.h index 951c16d5f..29472a250 100644 --- a/npu/vision_pack.h +++ b/npu/vision_pack.h @@ -69,6 +69,9 @@ struct Geometry { // and every row past it is zero in every block, else the tensors' own. A forward may stop at these; it must // still index the tensors at `intermediate` / `vit_merger_intermediate`. int model_intermediate = 0, model_vit_merger_intermediate = 0; + // The block after which the vit_merger runs: config.json's `insert_layer_id` (the reference's default, 6, when the + // config does not say). A value that is not a block of this tower is refused. + int insert_layer = 6; }; class Pack { diff --git a/npu/vision_tower.cpp b/npu/vision_tower.cpp new file mode 100644 index 000000000..20d359358 --- /dev/null +++ b/npu/vision_tower.cpp @@ -0,0 +1,232 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "vision_tower.h" + +#include +#include +#include +#include + +namespace onebit::npu::vision { + +namespace { + +// y[n, rows] = x[n, in] * W[rows_stored.., in_stride]^T + b, using the first `rows` rows and the first `in` columns of +// a weight stored `in_stride` wide. The padded MLPs are read through this: the stride is the tensor's, the extent the +// model's. Eight partial sums so the compiler can vectorise without being allowed to reorder a single chain. +void linear(const float* x, int n, int in, const float* w, int in_stride, const float* b, int rows, float* y) { + for (int t = 0; t < n; ++t) { + const float* xr = x + size_t(t) * in; + float* yr = y + size_t(t) * rows; + for (int r = 0; r < rows; ++r) { + const float* wr = w + size_t(r) * in_stride; + float acc[8] = {0, 0, 0, 0, 0, 0, 0, 0}; + int i = 0; + for (; i + 8 <= in; i += 8) + for (int k = 0; k < 8; ++k) acc[k] += xr[i + k] * wr[i + k]; + float s = ((acc[0] + acc[1]) + (acc[2] + acc[3])) + ((acc[4] + acc[5]) + (acc[6] + acc[7])); + for (; i < in; ++i) s += xr[i] * wr[i]; + yr[r] = s + (b ? b[r] : 0.0f); + } + } +} + +// torch.nn.LayerNorm: the population variance, eps inside the root. +void layer_norm(const float* x, int n, int width, const float* w, const float* b, float* y) { + for (int t = 0; t < n; ++t) { + const float* xr = x + size_t(t) * width; + float* yr = y + size_t(t) * width; + double mean = 0; + for (int i = 0; i < width; ++i) mean += xr[i]; + mean /= width; + double var = 0; + for (int i = 0; i < width; ++i) var += (xr[i] - mean) * (xr[i] - mean); + const float inv = float(1.0 / std::sqrt(var / width + 1e-6)); + for (int i = 0; i < width; ++i) yr[i] = float(xr[i] - mean) * inv * w[i] + b[i]; + } +} + +inline float gelu_tanh(float v) { + return 0.5f * v * (1.0f + std::tanh(0.7978845608028654f * (v + 0.044715f * v * v * v))); +} +inline float gelu_erf(float v) { return 0.5f * v * (1.0f + std::erf(v * 0.7071067811865476f)); } + +struct Attn { + std::vector qkv_w, qkv_b, proj_w, proj_b; +}; +Attn load_attn(const Pack& p, const std::string& stem) { + return {p.f32(stem + ".attn.qkv.weight"), p.f32(stem + ".attn.qkv.bias"), p.f32(stem + ".attn.proj.weight"), + p.f32(stem + ".attn.proj.bias")}; +} + +// out[n, H] = proj(attention(x)), where tokens attend only inside consecutive groups of `group` tokens (group == n is +// full attention). qkv is [3H, H]: q, k, v stacked in that order, heads contiguous inside each. +void attention(const float* x, int n, int H, int heads, int head_dim, int group, const Attn& a, float* out) { + std::vector qkv(size_t(n) * 3 * H), ctx(size_t(n) * H), prob(static_cast(group)); + linear(x, n, H, a.qkv_w.data(), H, a.qkv_b.data(), 3 * H, qkv.data()); + const float scale = 1.0f / std::sqrt(float(head_dim)); + for (int g0 = 0; g0 < n; g0 += group) { + for (int h = 0; h < heads; ++h) { + for (int i = 0; i < group; ++i) { + const float* q = qkv.data() + size_t(g0 + i) * 3 * H + size_t(h) * head_dim; + float mx = -INFINITY; + for (int j = 0; j < group; ++j) { + const float* k = qkv.data() + size_t(g0 + j) * 3 * H + H + size_t(h) * head_dim; + float acc[8] = {0, 0, 0, 0, 0, 0, 0, 0}; + int d = 0; + for (; d + 8 <= head_dim; d += 8) + for (int u = 0; u < 8; ++u) acc[u] += q[d + u] * k[d + u]; + float s = ((acc[0] + acc[1]) + (acc[2] + acc[3])) + ((acc[4] + acc[5]) + (acc[6] + acc[7])); + for (; d < head_dim; ++d) s += q[d] * k[d]; + prob[size_t(j)] = s * scale; + if (prob[size_t(j)] > mx) mx = prob[size_t(j)]; + } + double sum = 0; + for (int j = 0; j < group; ++j) { + prob[size_t(j)] = std::exp(prob[size_t(j)] - mx); + sum += prob[size_t(j)]; + } + float* o = ctx.data() + size_t(g0 + i) * H + size_t(h) * head_dim; + std::memset(o, 0, sizeof(float) * size_t(head_dim)); + for (int j = 0; j < group; ++j) { + const float pj = float(prob[size_t(j)] / sum); + const float* v = qkv.data() + size_t(g0 + j) * 3 * H + 2 * size_t(H) + size_t(h) * head_dim; + for (int d = 0; d < head_dim; ++d) o[d] += pj * v[d]; + } + } + } + } + linear(ctx.data(), n, H, a.proj_w.data(), H, a.proj_b.data(), H, out); +} + +// fc2(act(fc1(x))): `mid` rows of fc1 and `mid` columns of an fc2 stored `mid_stored` wide. +template +void mlp(const float* x, int n, int in, const std::vector& w1, const std::vector& b1, int mid, + const std::vector& w2, const std::vector& b2, int mid_stored, int out_w, Act act, float* out) { + std::vector h(size_t(n) * mid); + linear(x, n, in, w1.data(), in, b1.data(), mid, h.data()); + for (float& v : h) v = act(v); + linear(h.data(), n, mid, w2.data(), mid_stored, b2.data(), out_w, out); +} + +// The four tokens of each 2x2 window of a gh x gw row-major grid, side by side: out[(gh/2)*(gw/2), 4*H], windows +// row-major, inside a window (0,0) (0,1) (1,0) (1,1). This is both the vit_merger's window order and the merger's view. +std::vector windows(const float* x, int gh, int gw, int H) { + std::vector o(size_t(gh) * gw * H); + size_t at = 0; + for (int wy = 0; wy < gh / 2; ++wy) + for (int wx = 0; wx < gw / 2; ++wx) + for (int dy = 0; dy < 2; ++dy) + for (int dx = 0; dx < 2; ++dx) { + std::memcpy(o.data() + at, x + (size_t(2 * wy + dy) * gw + size_t(2 * wx + dx)) * H, sizeof(float) * size_t(H)); + at += size_t(H); + } + return o; +} + +} // namespace + +std::vector tower_forward(const Pack& p, const float* pixels, int gh, int gw, std::vector* tower) { + const Geometry& g = p.geometry(); + const std::string pre = "model.visual."; + if (g.heads <= 0 || g.head_dim <= 0) + throw std::runtime_error(p.path() + ": the head count is only in config.json, and none was given"); + if (g.insert_layer < 0 || g.insert_layer >= g.layers) + throw std::runtime_error(p.path() + ": the vit_merger runs after block " + std::to_string(g.insert_layer) + + ", which this tower of " + std::to_string(g.layers) + " blocks does not have"); + if (!pixels || gh <= 0 || gw <= 0 || gh % 4 != 0 || gw % 4 != 0) + throw std::runtime_error(p.path() + ": a " + std::to_string(gh) + " x " + std::to_string(gw) + + " patch grid cannot be merged twice by 2 x 2 (both extents must be multiples of 4)"); + if (gh > 4096 || gw > 4096) throw std::runtime_error(p.path() + ": the patch grid is larger than any image this tower takes"); + const int H = g.hidden, P = g.patch, C = g.channels, M = g.merge_hidden; + int n = gh * gw; + + // 1 + 2: patches (c, dy, dx) in the weight's own order, then the nearest position row + std::vector x(size_t(n) * H); + { + const std::vector w = p.f32(pre + "patch_embed.proj.weight"), b = p.f32(pre + "patch_embed.proj.bias"); + const std::vector pos = p.f32(pre + "pos_embed.weight"); + const int pd = C * P * P, W = gw * P; + std::vector patch(static_cast(pd)); + for (int y = 0; y < gh; ++y) + for (int xx = 0; xx < gw; ++xx) { + size_t at = 0; + for (int c = 0; c < C; ++c) + for (int dy = 0; dy < P; ++dy) + for (int dx = 0; dx < P; ++dx) + patch[at++] = pixels[(size_t(c) * gh * P + size_t(y * P + dy)) * W + size_t(xx * P + dx)]; + float* row = x.data() + (size_t(y) * gw + xx) * H; + linear(patch.data(), 1, pd, w.data(), pd, b.data(), H, row); + const float* pr = pos.data() + size_t(nearest_position(y, xx, gh, gw, g.grid)) * H; + for (int i = 0; i < H; ++i) row[i] += pr[i]; + } + } + + std::vector t(size_t(n) * H), u(size_t(n) * H); + const auto tanh_act = [](float v) { return gelu_tanh(v); }; + for (int layer = 0; layer < g.layers; ++layer) { + const std::string b = pre + "blocks." + std::to_string(layer); + // 3 / 5: one block + layer_norm(x.data(), n, H, p.f32(b + ".norm1.weight").data(), p.f32(b + ".norm1.bias").data(), t.data()); + attention(t.data(), n, H, g.heads, g.head_dim, n, load_attn(p, b), u.data()); + for (size_t i = 0; i < size_t(n) * H; ++i) x[i] += u[i]; + layer_norm(x.data(), n, H, p.f32(b + ".norm2.weight").data(), p.f32(b + ".norm2.bias").data(), t.data()); + mlp(t.data(), n, H, p.f32(b + ".mlp.linear_fc1.weight"), p.f32(b + ".mlp.linear_fc1.bias"), g.model_intermediate, + p.f32(b + ".mlp.linear_fc2.weight"), p.f32(b + ".mlp.linear_fc2.bias"), g.intermediate, H, tanh_act, u.data()); + for (size_t i = 0; i < size_t(n) * H; ++i) x[i] += u[i]; + + if (layer != g.insert_layer) continue; + // 4: the vit_merger + const std::string v = pre + "vit_merger"; + layer_norm(x.data(), n, H, p.f32(v + ".norm1.weight").data(), p.f32(v + ".norm1.bias").data(), t.data()); + const std::vector normed = windows(t.data(), gh, gw, H); + std::vector res = windows(x.data(), gh, gw, H); // the residual, in window order too + attention(normed.data(), n, H, g.heads, g.head_dim, 4, load_attn(p, v), u.data()); + for (size_t i = 0; i < size_t(n) * H; ++i) res[i] += u[i]; // (n/4 windows) x (4 tokens side by side) + const int nw = n / 4; + std::vector flat(size_t(nw) * M), y(size_t(nw) * H); + layer_norm(res.data(), nw, M, p.f32(v + ".pre_norm.weight").data(), p.f32(v + ".pre_norm.bias").data(), flat.data()); + mlp(flat.data(), nw, M, p.f32(v + ".mlp.linear_fc1.weight"), p.f32(v + ".mlp.linear_fc1.bias"), + g.model_vit_merger_intermediate, p.f32(v + ".mlp.linear_fc2.weight"), p.f32(v + ".mlp.linear_fc2.bias"), + g.vit_merger_intermediate, H, tanh_act, y.data()); + for (int wdw = 0; wdw < nw; ++wdw) + for (int i = 0; i < H; ++i) { + const float* r = res.data() + size_t(wdw) * M; + y[size_t(wdw) * H + i] += 0.25f * (r[i] + r[H + i] + r[2 * H + i] + r[3 * H + i]); + } + x.swap(y); + gh /= 2; + gw /= 2; + n = nw; + } + + // 5: the tower's last LayerNorm + std::vector last(size_t(n) * H); + layer_norm(x.data(), n, H, p.f32(pre + "merger.norm.weight").data(), p.f32(pre + "merger.norm.bias").data(), last.data()); + if (tower) *tower = last; + + // 6: the merger + const int no = n / 4; + const std::vector grouped = windows(last.data(), gh, gw, H); + std::vector normed(size_t(no) * M), out(size_t(no) * g.out_hidden); + layer_norm(grouped.data(), no, M, p.f32(pre + "merger.pre_norm.weight").data(), p.f32(pre + "merger.pre_norm.bias").data(), + normed.data()); + mlp(normed.data(), no, M, p.f32(pre + "merger.linear_fc1.weight"), p.f32(pre + "merger.linear_fc1.bias"), M, + p.f32(pre + "merger.linear_fc2.weight"), p.f32(pre + "merger.linear_fc2.bias"), M, g.out_hidden, + [](float v) { return gelu_erf(v); }, out.data()); + return out; +} + +} // namespace onebit::npu::vision diff --git a/npu/vision_tower.h b/npu/vision_tower.h new file mode 100644 index 000000000..79cbe2167 --- /dev/null +++ b/npu/vision_tower.h @@ -0,0 +1,54 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// The MiniCPM-V 4.7 vision tower, host float32 (engine#1788): pixels in, the rows the text model receives out. +// +// A reference, not a serving path: plain loops, one image, no device. It exists so that whatever later runs this +// tower on the NPU has numbers to be compared with, and it is itself compared with the transformers implementation +// (transformers.models.minicpmv4_7, 5.22.2) on a published pack's weights: docs/npu.md has the figures. +// +// The order of operations, each line read from that implementation and pinned by tests/npu_vision_tower_test.cpp: +// 1. patch embedding: a Conv2d whose kernel is its stride, so one matrix product per patch, patches row-major; +// 2. + a learned position row per patch, picked NEAREST on the pack's square table: row floor(y * side / gh), +// column floor(x * side / gw). No interpolation. (The reference buckets y/gh in float32 against k/side; the integer +// form gives the same index for every grid height and width from 1 to 140 at side 70, measured.) +// 3. blocks 0..insert_layer: x += proj(attention(LN1(x))), x += fc2(gelu_tanh(fc1(LN2(x)))). Full attention over +// the image, no rotary term, scale 1/sqrt(head_dim), LayerNorm eps 1e-6; +// 4. the vit_merger, once, after block insert_layer: LN1, attention INSIDE each 2x2 window only, the residual, then +// per window fc2(gelu_tanh(fc1(pre_norm(the four tokens side by side)))) + the mean of the four. The grid halves; +// 5. the remaining blocks on the halved grid, then one LayerNorm (the pack's `merger.norm`); +// 6. the merger: 2x2 groups again, pre_norm over the four tokens side by side, fc1, GELU (the exact erf form here, +// not the tanh approximation the blocks use), fc2 to the text model's width. The grid halves again. +// So a gh x gw patch grid gives (gh/4) * (gw/4) rows, row-major, and both extents must be multiples of 4. +#pragma once + +#include "vision_pack.h" + +#include + +namespace onebit::npu::vision { + +// `pixels` is (channels, gh*patch, gw*patch) float32, already rescaled and normalised. Returns (gh/4 * gw/4, +// out_hidden). When `tower` is given it receives step 5's output, (gh/2 * gw/2, hidden): the tower before the merger. +// Throws std::runtime_error when the grid is not a multiple of 4 in both extents, is empty, or the pack's geometry +// lacks what only config.json states (the head count). +std::vector tower_forward(const Pack& pack, const float* pixels, int gh, int gw, std::vector* tower = nullptr); + +// Step 2's table row for patch (y, x) of a gh x gw grid on a side x side table. +inline int nearest_position(int y, int x, int gh, int gw, int side) { + return int((int64_t(y) * side) / gh) * side + int((int64_t(x) * side) / gw); +} + +} // namespace onebit::npu::vision diff --git a/tests/npu_vision_pack_test.cpp b/tests/npu_vision_pack_test.cpp index 9d2fbeb5c..fd260df74 100644 --- a/tests/npu_vision_pack_test.cpp +++ b/tests/npu_vision_pack_test.cpp @@ -378,6 +378,22 @@ void test_refusals() { refused("config: another patch size", with("PATCH_SIZE", 4), {"PATCH_SIZE 4"}); refused("config: an image the position table does not cover", with("IMAGE_SIZE", 12), {"IMAGE_SIZE 12", "4 x 4"}); refused("config: another position count", with("VISION_NUM_POSITION_EMBEDDINGS", 25), {"NUM_POSITION_EMBEDDINGS 25", "16 rows"}); + // the vit_merger's place: read from the config, refused when it is not a block or is stated twice differently + { + json c = config_of(s); + c["insert_layer_id"] = 1; + try { + const vis::Pack p(kPath, c.dump()); + check(p.geometry().insert_layer == 1, "config: insert_layer_id is read"); + } catch (const std::exception& e) { + check(false, std::string("a config with insert_layer_id loads (threw: ") + e.what() + ")"); + } + c["insert_layer_id"] = 2; + refused("config: a merge point past the last block", c.dump(), {"insert_layer_id 2", "2 blocks"}); + c["insert_layer_id"] = 0; + c["vision_config"]["MINICPM_V_INSERT_LAYER_ID"] = 1; + refused("config: two merge points", c.dump(), {"insert_layer_id 0", "MINICPM_V_INSERT_LAYER_ID 1"}); + } { json c = config_of(s); c["hidden_size"] = 12; @@ -425,6 +441,7 @@ void test_published_pack(const std::string& dir) { check(g.model_intermediate == 4304 && g.model_vit_merger_intermediate == 17216, "published pack: the model's MLP widths are 4304 and 17216, the rest verified zero in all 27 blocks"); check(g.merge_hidden == 4608, "published pack: mergers at 4 x 1152"); + check(g.insert_layer == 6, "published pack: the vit_merger runs after block 6"); check(p.notes().size() == 2, "published pack: both padded widths are noted"); // The 1B pack's container names its source; the 35B-A3B's carries no __metadata__ at all. Both are accepted, // and a source hash that is present has to look like one. diff --git a/tests/npu_vision_tower_test.cpp b/tests/npu_vision_tower_test.cpp new file mode 100644 index 000000000..fcb71588a --- /dev/null +++ b/tests/npu_vision_tower_test.cpp @@ -0,0 +1,302 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// npu_vision_tower_test +// +// The host float32 MiniCPM-V 4.7 vision tower (npu/vision_tower.h, engine#1788) against the transformers +// implementation (transformers.models.minicpmv4_7, 5.22.2, torch 2.13 CPU, float32). +// +// The fixture is a small tower written here: width 16 (2 heads x 8), 2-pixel patches, a 4 x 4 position table, 8 blocks +// with the vit_merger after block 6, MLPs 12 wide stored 16 wide (zero padded, as the published packs store theirs), +// projecting to 12. Its weights come from a 32-bit LCG, so the same file can be rebuilt anywhere; the goldens below are +// what the transformers classes return for that file and the input pixel(c, y, x) = sin(0.37c + 0.11y + 0.05x + +// 0.003xy) on a 4 x 8 patch grid. Nothing here was produced by the code under test. +// +// ONEBIT_VISION_KEEP_FIXTURE=1 leaves the container and its config.json in the working directory (that is how the +// goldens were produced). ONEBIT_MINICPM_PACK= with ONEBIT_MINICPM_ORACLE= compares a published pack's output with a reference run of the same input. +#include "../npu/vision_tower.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using nlohmann::json; +namespace vis = onebit::npu::vision; + +int g_fail = 0, g_arms = 0; +void check(bool ok, const std::string& what) { + ++g_arms; + if (!ok) { + ++g_fail; + std::printf("FAIL %s\n", what.c_str()); + } +} + +const int H = 16, C = 3, P = 2, SIDE = 4, L = 8, I = 12, I_STORED = 16, VI = 48, VI_STORED = 64, O = 12; +const int GH = 4, GW = 8; + +uint32_t g_lcg = 20261110u; +float lcg_unit() { // in [-1, 1) + g_lcg = g_lcg * 1664525u + 1013904223u; + return float(g_lcg >> 8) / 8388608.0f - 1.0f; +} +uint16_t bf16(float v) { + uint32_t b; + std::memcpy(&b, &v, 4); + return uint16_t((b + 0x8000u) >> 16); +} + +struct Writer { + json table = json::object(); + std::vector payload; + // kind 0: a matrix or bias, 0.3 * u (bias 0.1 * u); kind 1: a norm weight, 1 + 0.2 * u. + // Elements at or past `live` along `axis` are zero: the container's padding. + void add(const std::string& name, std::vector shape, int kind, int axis = -1, int64_t live = 0) { + uint64_t n = 1; + for (int64_t d : shape) n *= uint64_t(d); + const uint64_t off = payload.size(); + const int64_t cols = shape.size() > 1 ? shape[1] : 1; + for (uint64_t e = 0; e < n; ++e) { + const bool is_bias = shape.size() == 1 && kind == 0; + float v = kind == 1 ? 1.0f + 0.2f * lcg_unit() : (is_bias ? 0.1f : 0.3f) * lcg_unit(); + if (axis == 0 && int64_t(e) / cols >= live) v = 0.0f; + if (axis == 1 && int64_t(e) % cols >= live) v = 0.0f; + const uint16_t h = bf16(v); + payload.push_back(uint8_t(h & 0xff)); + payload.push_back(uint8_t(h >> 8)); + } + table["model.visual." + name] = {{"dtype", "BF16"}, {"shape", shape}, {"data_offsets", {off, off + n * 2}}}; + } + void linear(const std::string& stem, int64_t rows, int64_t cols) { + add(stem + ".weight", {rows, cols}, 0); + add(stem + ".bias", {rows}, 0); + } + void norm(const std::string& stem, int64_t n) { + add(stem + ".weight", {n}, 1); + add(stem + ".bias", {n}, 0); + } + void padded_mlp(const std::string& stem, int64_t in, int64_t live, int64_t stored, int64_t out) { + add(stem + ".linear_fc1.weight", {stored, in}, 0, 0, live); + add(stem + ".linear_fc1.bias", {stored}, 0, 0, live); + add(stem + ".linear_fc2.weight", {out, stored}, 0, 1, live); + add(stem + ".linear_fc2.bias", {out}, 0); + } +}; + +const char* const kContainer = "npu_vision_tower_test.q4nx"; +const char* const kConfig = "npu_vision_tower_test.config.json"; + +std::string write_fixture() { + g_lcg = 20261110u; + Writer w; + w.add("patch_embed.proj.weight", {H, C, P, P}, 0); + w.add("patch_embed.proj.bias", {H}, 0); + w.add("pos_embed.weight", {SIDE * SIDE, H}, 0); + for (int i = 0; i < L; ++i) { + const std::string b = "blocks." + std::to_string(i) + "."; + w.norm(b + "norm1", H); + w.linear(b + "attn.qkv", 3 * H, H); + w.linear(b + "attn.proj", H, H); + w.norm(b + "norm2", H); + w.padded_mlp(b + "mlp", H, I, I_STORED, H); + } + w.norm("vit_merger.norm1", H); + w.linear("vit_merger.attn.qkv", 3 * H, H); + w.linear("vit_merger.attn.proj", H, H); + w.norm("vit_merger.pre_norm", 4 * H); + w.padded_mlp("vit_merger.mlp", 4 * H, VI, VI_STORED, H); + w.norm("merger.norm", H); + w.norm("merger.pre_norm", 4 * H); + w.linear("merger.linear_fc1", 4 * H, 4 * H); + w.linear("merger.linear_fc2", O, 4 * H); + const std::string head = w.table.dump(); + const uint64_t len = head.size(); + std::ofstream f(kContainer, std::ios::binary | std::ios::trunc); + f.write(reinterpret_cast(&len), 8); + f.write(head.data(), std::streamsize(head.size())); + f.write(reinterpret_cast(w.payload.data()), std::streamsize(w.payload.size())); + + json vc; + vc["MINICPM_V_PATCH_SIZE"] = P; + vc["MINICPM_V_IMAGE_SIZE"] = SIDE * P; + vc["MINICPM_V_VISION_EMBED_DIM"] = H; + vc["MINICPM_V_VISION_NUM_HEADS"] = 2; + vc["MINICPM_V_VISION_HEAD_DIM"] = 8; + vc["MINICPM_V_VISION_MLP_INTERMEDIATE_SIZE"] = I; + vc["MINICPM_V_VISION_NUM_POSITION_EMBEDDINGS"] = SIDE * SIDE; + vc["MINICPM_V_VISION_NUM_LAYERS"] = L; + vc["MINICPM_V_VISION_LAYER_NORM_EPSILON"] = 1e-6; + vc["MINICPM_V_VIT_MERGER_INTERMEDIATE_SIZE"] = VI; + json c; + c["hidden_size"] = O; + c["insert_layer_id"] = 6; + c["vision_config"] = vc; + const std::string text = c.dump(); + std::ofstream cf(kConfig); + cf << text; + return text; +} + +std::vector test_pixels(int gh, int gw, int patch) { + const int hp = gh * patch, wp = gw * patch; + std::vector px(size_t(3) * hp * wp); + for (int c = 0; c < 3; ++c) + for (int y = 0; y < hp; ++y) + for (int x = 0; x < wp; ++x) + px[(size_t(c) * hp + y) * wp + x] = float(std::sin(0.37 * c + 0.11 * y + 0.05 * x + 0.003 * double(x) * y)); + return px; +} + +// GOLDENS_BEGIN (transformers 5.22.2, torch 2.13.0+cpu, float32, eager attention) +const float kGoldenOut[2 * O] = { + 1.66476488f, -0.92423868f, 0.45880169f, 0.54087150f, 0.91351569f, -2.78720903f, + 1.69952393f, 0.54694951f, -1.22599459f, 2.31425953f, 0.93287897f, 0.15626144f, + 1.68985367f, -0.91641307f, 0.10820520f, 0.28901398f, 0.99814737f, -2.51858306f, + 1.39385486f, 0.41435477f, -1.09378004f, 2.35683727f, 0.77005690f, 0.12177789f, +}; +const double kGoldenTowerSum = -0.859797; +// GOLDENS_END + +double max_abs_diff(const std::vector& a, const float* b, size_t n) { + double m = a.size() == n ? 0.0 : 1e30; + for (size_t i = 0; i < n && i < a.size(); ++i) m = std::fmax(m, std::fabs(double(a[i]) - double(b[i]))); + return m; +} + +void test_fixture(const std::string& cfg) { + try { + const vis::Pack p(kContainer, cfg); + check(p.geometry().model_intermediate == I && p.geometry().intermediate == I_STORED && p.geometry().insert_layer == 6, + "fixture: MLPs 12 wide stored 16 wide, vit_merger after block 6"); + const std::vector px = test_pixels(GH, GW, P); + std::vector tower; + const std::vector out = vis::tower_forward(p, px.data(), GH, GW, &tower); + check(out.size() == size_t(2 * O) && tower.size() == size_t(GH / 2) * (GW / 2) * H, + "fixture: a 4 x 8 grid gives 8 tower rows and 2 output rows"); + const double d = max_abs_diff(out, kGoldenOut, size_t(2 * O)); + double sum = 0; + for (float v : tower) sum += v; + std::printf(" fixture: max |out - reference| = %.3g; tower sum %.6f (reference %.6f)\n", d, sum, kGoldenTowerSum); + std::printf(" out[0,:6] %.8f %.8f %.8f %.8f %.8f %.8f\n", out[0], out[1], out[2], out[3], out[4], out[5]); + check(d < 2e-5, "fixture: every output value is the reference's within 2e-5"); + check(std::fabs(sum - kGoldenTowerSum) < 2e-3, "fixture: the tower before the merger sums to the reference's"); + + // grids the two 2 x 2 merges cannot take + for (const auto& [gh, gw] : std::vector>{{4, 6}, {2, 8}, {0, 8}, {4, -4}}) { + std::string why; + try { vis::tower_forward(p, px.data(), gh, gw); } catch (const std::exception& e) { why = e.what(); } + check(why.find("multiples of 4") != std::string::npos, + "a " + std::to_string(gh) + " x " + std::to_string(gw) + " grid is refused by name"); + } + } catch (const std::exception& e) { + check(false, std::string("the fixture runs (threw: ") + e.what() + ")"); + } + // without a config there is no head count, and the forward says so instead of guessing one + try { + const vis::Pack p(kContainer, ""); + const std::vector px = test_pixels(GH, GW, P); + std::string why; + try { vis::tower_forward(p, px.data(), GH, GW); } catch (const std::exception& e) { why = e.what(); } + check(why.find("head count") != std::string::npos, "no config: the forward refuses for want of a head count"); + } catch (const std::exception& e) { + check(false, std::string("the fixture opens without a config (threw: ") + e.what() + ")"); + } +} + +// The reference buckets y / gh (float32) against k / side; the integer form must give the same row and column. These +// are the reference's own indices for a 70-wide table (printed from get_vision_nearest_position_ids). +void test_nearest_position() { + check(vis::nearest_position(0, 0, 8, 12, 70) == 0, "nearest position: the first patch is table row 0"); + check(vis::nearest_position(7, 11, 8, 12, 70) == 61 * 70 + 64, "nearest position: (7, 11) of 8 x 12 is (61, 64)"); + check(vis::nearest_position(3, 5, 8, 12, 70) == 26 * 70 + 29, "nearest position: (3, 5) of 8 x 12 is (26, 29)"); + check(vis::nearest_position(69, 69, 70, 70, 70) == 69 * 70 + 69, "nearest position: a full grid maps onto itself"); + check(vis::nearest_position(139, 0, 140, 4, 70) == 69 * 70, "nearest position: a grid taller than the table shares rows"); +} + +std::vector read_f32(const std::string& path) { + std::ifstream f(path, std::ios::binary | std::ios::ate); + if (!f) return {}; + std::vector v(size_t(f.tellg()) / 4); + f.seekg(0); + f.read(reinterpret_cast(v.data()), std::streamsize(v.size() * 4)); + return v; +} + +void test_published(const std::string& pack_dir, const std::string& oracle) { + try { + int gh = 0, gw = 0; + { std::ifstream g(oracle + "/grid.txt"); g >> gh >> gw; } + const std::vector px = read_f32(oracle + "/pixels.f32"), want = read_f32(oracle + "/out.f32"), + want_tower = read_f32(oracle + "/tower.f32"); + const vis::Pack p(pack_dir); + check(gh > 0 && gw > 0 && px.size() == size_t(3) * gh * gw * p.geometry().patch * p.geometry().patch, + "published pack: the reference input is this grid's pixels"); + std::vector tower; + const std::vector out = vis::tower_forward(p, px.data(), gh, gw, &tower); + const auto stats = [](const std::vector& a, const std::vector& b, double* mx, double* corr, double* mean_abs) { + double sa = 0, sb = 0, saa = 0, sbb = 0, sab = 0; + *mx = a.size() == b.size() ? 0 : 1e30; + *mean_abs = 0; + const size_t n = a.size() < b.size() ? a.size() : b.size(); + for (size_t i = 0; i < n; ++i) { + *mx = std::fmax(*mx, std::fabs(double(a[i]) - b[i])); + sa += a[i]; sb += b[i]; saa += double(a[i]) * a[i]; sbb += double(b[i]) * b[i]; sab += double(a[i]) * b[i]; + *mean_abs += std::fabs(double(b[i])); + } + *mean_abs /= double(n ? n : 1); + const double va = saa - sa * sa / double(n), vb = sbb - sb * sb / double(n); + *corr = (sab - sa * sb / double(n)) / std::sqrt(va * vb); + }; + double mx, corr, mean_abs; + stats(tower, want_tower, &mx, &corr, &mean_abs); + std::printf(" published pack, %d x %d grid: tower %zu values, max |d| %.3g, corr %.9f, reference mean |v| %.4f\n", gh, gw, + tower.size(), mx, corr, mean_abs); + check(tower.size() == want_tower.size() && corr > 0.99999, "published pack: the tower before the merger matches the reference"); + stats(out, want, &mx, &corr, &mean_abs); + std::printf(" published pack, %d x %d grid: output %zu values, max |d| %.3g, corr %.9f, reference mean |v| %.4f\n", gh, gw, + out.size(), mx, corr, mean_abs); + check(out.size() == want.size() && corr > 0.99999 && mx < 5e-3, "published pack: the output matches the reference"); + } catch (const std::exception& e) { + check(false, std::string("the published pack runs (threw: ") + e.what() + ")"); + } +} + +} // namespace + +int main() { + const std::string cfg = write_fixture(); + test_fixture(cfg); + test_nearest_position(); + if (!std::getenv("ONEBIT_VISION_KEEP_FIXTURE")) { + std::remove(kContainer); + std::remove(kConfig); + } + const char* pack = std::getenv("ONEBIT_MINICPM_PACK"); + const char* oracle = std::getenv("ONEBIT_MINICPM_ORACLE"); + if (pack && *pack && oracle && *oracle) test_published(pack, oracle); + else std::printf(" (ONEBIT_MINICPM_PACK / ONEBIT_MINICPM_ORACLE not set: the published-pack arm did not run)\n"); + std::printf("vision tower: %d checks, %d failed\n", g_arms, g_fail); + return g_fail ? 1 : 0; +}