diff --git a/CMakeLists.txt b/CMakeLists.txt index 1b5dbe682..913000720 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -687,7 +687,7 @@ option(ONEBIT_NPU_HOST "Build the NPU host library and its tests" ${_onebit_npu_ if(ONEBIT_NPU_HOST) find_package(PkgConfig REQUIRED) pkg_check_modules(PCRE2 REQUIRED IMPORTED_TARGET libpcre2-8) -add_library(onebit_npu STATIC npu/file_map.cpp npu/flm_head_import.cpp npu/full_elf.cpp npu/model.cpp +add_library(onebit_npu STATIC npu/file_map.cpp npu/flm_head_import.cpp npu/full_elf.cpp npu/model.cpp npu/vision_pack.cpp npu/moe.cpp npu/pack.cpp npu/pack_import.cpp npu/private_route.cpp npu/q4nx.cpp npu/q4nx_device.cpp npu/tokenizer.cpp) target_include_directories(onebit_npu PUBLIC npu) @@ -1194,6 +1194,12 @@ if(ONEBIT_NPU_CAPTURED AND ONEBIT_NPU_PDI AND ONEBIT_NPU_GOLDEN) endif() # Q4NX chunk decoders against 1bit-MONSTER's verified decoders, on committed real tiles +# engine#1788: the vision component reader (vision_weight.q4nx). Crafted containers for every refusal; the published +# pack's own numbers when ONEBIT_MINICPM_PACK names a MiniCPM-V 4.7 pack directory. +add_executable(npu_vision_pack_test tests/npu_vision_pack_test.cpp) +target_link_libraries(npu_vision_pack_test PRIVATE onebit_npu) +add_test(NAME npu_vision_pack COMMAND npu_vision_pack_test) +add_dependencies(onebit npu_vision_pack_test) add_executable(npu_q4nx_test tests/npu_q4nx_test.cpp) target_include_directories(npu_q4nx_test PRIVATE app) target_link_libraries(npu_q4nx_test PRIVATE onebit_npu) diff --git a/app/main.cpp b/app/main.cpp index e9249b378..1d7ae3755 100644 --- a/app/main.cpp +++ b/app/main.cpp @@ -43,6 +43,7 @@ int onebit_ort_run_main(int argc, char** argv); #endif #ifdef ONEBIT_NPU_HOST #include "pack_import.h" // 1bit npu-pack: an OGA NPU pack -> a Q4NX model directory +#include "vision_pack.h" // 1bit npu-pack vision: a pack's vision component (engine#1788) #endif #ifdef ONEBIT_NPU #include "forward_serve.h" @@ -94,6 +95,7 @@ void usage(FILE* out) { #endif #ifdef ONEBIT_NPU_HOST " npu-pack import a Ryzen AI OGA NPU pack (npu-pack --help)\n" + " npu-pack vision read a pack's vision component (vision_weight.q4nx)\n" #endif " comfy run a ComfyUI workflow with ComfyUI.cpp (docs/comfyui.md)\n" #ifdef ONEBIT_LAYA @@ -323,6 +325,18 @@ int main(int argc, char** argv) { #ifdef ONEBIT_NPU_HOST if (cmd == "npu-pack") { try { + // `npu-pack vision `: the vision component of a two-component pack (engine#1788), read by + // npu/vision_pack.h rather than the OGA importer, which knows nothing of it. + if (argc >= 3 && std::string(argv[2]) == "vision") { + if (argc != 4 || argv[3][0] == '-') { + std::fprintf(stderr, "usage: 1bit npu-pack vision \n" + " reads the pack's vision component (config.json's vision_model_weight, default\n" + " vision_weight.q4nx): every tensor placed, the tower's geometry from the tensors'\n" + " own shapes, and each config.json field the tensors overrule\n"); + return argc == 4 ? 0 : 2; + } + return onebit::npu::vision::print_info(argv[3]); + } return onebit::npu::pack_import::run(argc - 2, argv + 2); } catch (const std::exception& e) { std::fprintf(stderr, "1bit npu-pack: %s\n", e.what()); diff --git a/docs/npu.md b/docs/npu.md index aa2bcb1bc..cdc4c9dd5 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1448,6 +1448,66 @@ selections per layer touch nearly all 128 experts and stream ~18 GB. MoE prefill GPU route (`--moe-prefetch`, docs/serve.md); this is the thing that decides whether a kernel is right. +### A pack's vision component (`vision_weight.q4nx`, engine#1788) + +FastFlowLM's MiniCPM-V 4.7 packs are two containers at the pack root: `model.q4nx` (the text model) and +the file `config.json` names as `vision_model_weight`, `vision_weight.q4nx` (the vision tower). The +second is the same container format holding a ViT: every tensor BF16, no tiles. `npu/vision_pack.h` +reads it, and `1bit npu-pack vision ` prints what it found: + +``` +$ 1bit npu-pack vision MiniCPM-V-4.7-1B-NPU2 +vision component MiniCPM-V-4.7-1B-NPU2/vision_weight.q4nx +tensors 347, all BF16, 1135109120 B of payload, 0 B unaccounted +patch embed 14 x 14 pixels x 3 channels -> 1152 +positions 4900 (a 70 x 70 grid; image 980) +blocks 27 at width 1152, 16 heads x 72, MLP 4304 (4608 stored) +vit_merger attention at 1152, MLP 4608 -> 17216 (17280 stored) -> 1152 +merger 4608 -> 4608 -> 1024 (the text model's hidden size) +source_file mmproj-model-f16.gguf +source_sha256 fb9cc047a3cdad5317fd8433a4aeda2d22f00f02ca4bd93b34ad84165b25a900 +note config.json states MINICPM_V_VISION_MLP_INTERMEDIATE_SIZE 4304; model.visual.blocks.*.mlp.linear_fc1.weight has 4608 rows, the last 304 all zero: the container pads the model's width, and sizes come from the tensor +note config.json states MINICPM_V_VIT_MERGER_INTERMEDIATE_SIZE 17216; model.visual.vit_merger.mlp.linear_fc1.weight has 17280 rows, the last 64 all zero: the container pads the model's width, and sizes come from the tensor +``` + +This is a reader, not a forward: nothing runs the tower yet. What it settles is what a forward may +rely on. + +- **Sizes come from the tensors.** The geometry is derived from the tensors' own shapes, and every + tensor must have the shape the others imply. +- **The MLPs are stored padded.** The model's MLP widths are 4304 (blocks) and 17216 (`vit_merger`); + that is what `config.json` states and what the source GGUF's tensors are. The container stores them + 4608 and 17280 wide, the next multiples of 128, with the extra `linear_fc1` rows, bias entries and + `linear_fc2` columns all zero. A loader that sized a buffer from the config would be 304 rows short + of what the file holds. The reader sizes from the tensor, verifies that everything past the stated + width is zero in every block, and carries the model's width beside the stored one + (`Geometry::model_intermediate`). If any value past the stated width is not zero it says so and + keeps the tensor's width; a config that states an MLP wider than its tensors is refused. +- **Zero gaps.** A tensor the reader has no place for is a refusal, as is a missing one, a hole in the + block indices, a tensor that is not BF16, an extent that is not its shape's byte length, an extent + outside the payload, and two tensors on the same bytes. Each names the tensor. +- **`config.json` is consulted for what the tensors cannot say** (the head count, the image size, the + unpadded MLP widths) and refused where the two cannot both be right: heads that do not multiply to + the width, an image size that is not the position grid times the patch, another width, block count + or position count, and a text `hidden_size` that is not what the merger projects to. That last one + is a tower paired with the wrong text model. + +**Checked against the source, bit for bit** (2026-11-10, both published packs). The 1B container's +`__metadata__` names `mmproj-model-f16.gguf` and its SHA-256, which is the file of that name in +`openbmb/MiniCPM-V-4.7-1B-gguf`. All 347 pack tensors were rebuilt from that GGUF and compared as +BF16 bit patterns: 263 identical, 84 identical in the source's extent and `+0.0` in the padding, 0 +different. The pack's `attn.qkv` is the source's `attn_q`, `attn_k`, `attn_v` stacked in that order; +`merger.norm` is the source's `v.post_ln`. Every F32 source tensor is exactly representable in BF16. +The 35B-A3B container carries no `__metadata__`; the same comparison against +`openbmb/MiniCPM-V-4.7-35B-A3B-gguf`'s `mmproj-model-f16.gguf` (SHA-256 `bff3b688…5d20f1`) gives the +same 263 / 84 / 0, and against the 1B's source gives 0 of 347, so the two towers share a geometry and +not a single tensor. One source tensor has no counterpart in either container: `v.tok_embd_sep` +(`[4, hidden]`). + +Pinned by `tests/npu_vision_pack_test.cpp` on crafted containers; `ONEBIT_MINICPM_PACK=` adds +the published pack's own numbers. The source comparison above was run by hand and is not a test in the +tree. + ## Serving The 1bit NPU engine runs the model on the XDNA 2 NPU. With a release package that carries the NPU diff --git a/npu/vision_pack.cpp b/npu/vision_pack.cpp new file mode 100644 index 000000000..4304b5c69 --- /dev/null +++ b/npu/vision_pack.cpp @@ -0,0 +1,394 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "vision_pack.h" + +#include "file_map.h" + +#include + +#include +#include +#include +#include +#include +#include + +namespace onebit::npu::vision { + +namespace { + +using nlohmann::json; +using Shape = std::vector; + +const char* const kPrefix = "model.visual."; + +std::string shape_text(const Shape& s) { + std::string o = "["; + for (size_t i = 0; i < s.size(); ++i) o += (i ? ", " : "") + std::to_string(s[i]); + return o + "]"; +} + +// The product of a shape, or false when a dimension is not positive or the product does not fit: a shape is file +// content, and its product sizes a read. +bool elements_of(const Shape& s, uint64_t* out) { + uint64_t n = 1; + if (s.empty()) return false; + for (int64_t d : s) { + if (d <= 0) return false; + if (n > (uint64_t(1) << 40) / uint64_t(d)) return false; + n *= uint64_t(d); + } + *out = n; + return true; +} + +int as_int(int64_t v) { return v > 0 && v <= (int64_t(1) << 30) ? int(v) : 0; } + +} // namespace + +Pack::Pack(const std::string& pack_dir) { + std::ifstream cf(pack_dir + "/config.json"); + if (!cf) throw std::runtime_error("cannot read " + pack_dir + "/config.json"); + std::stringstream ss; + ss << cf.rdbuf(); + const std::string text = ss.str(); + std::string name = "vision_weight.q4nx"; + const json c = json::parse(text, nullptr, false); + if (c.is_discarded() || !c.is_object()) throw std::runtime_error(pack_dir + "/config.json is not a JSON object"); + if (c.contains("vision_model_weight") && c["vision_model_weight"].is_string()) + name = c["vision_model_weight"].get(); + // The name is file content and is joined to a directory: a separator in it would leave the pack. + if (name.empty() || name.find('/') != std::string::npos || name.find('\\') != std::string::npos || name == "..") + throw std::runtime_error(pack_dir + "/config.json: vision_model_weight \"" + name + "\" is not a file name"); + open(pack_dir + "/" + name, text); +} + +Pack::Pack(const std::string& container_path, const std::string& config_json) { open(container_path, config_json); } + +Pack::~Pack() { + if (map_) unmap_file(map_, size_); +} + +void Pack::open(const std::string& container_path, const std::string& config_json) { + path_ = container_path; + const auto fail = [&](const std::string& what) -> void { throw std::runtime_error(path_ + ": " + what); }; + + std::string why; + map_ = map_file(path_, size_, &why); + if (!map_) throw std::runtime_error(why); + if (size_ < 8) fail("truncated (no header length)"); + uint64_t header = 0; + std::memcpy(&header, map_, 8); + if (header > size_ - 8) fail("the header runs past the end of the file"); + const json table = json::parse(map_ + 8, map_ + 8 + header, nullptr, false); + if (table.is_discarded() || !table.is_object()) fail("the header is not a JSON object"); + payload_ = size_ - 8 - header; + const uint8_t* base = map_ + 8 + header; + + // ── every tensor: BF16, inside the payload, exactly as long as its shape says ── + for (const auto& [name, t] : table.items()) { + if (name == "__metadata__") { + if (t.is_object()) + for (const auto& [k, v] : t.items()) meta_[k] = v.is_string() ? v.get() : v.dump(); + continue; + } + if (!t.is_object() || !t.contains("data_offsets") || !t.contains("shape") || !t.contains("dtype")) + fail("entry " + name + " is not a tensor record (dtype, shape, data_offsets)"); + if (!t["dtype"].is_string() || t["dtype"].get() != "BF16") + fail(name + " is " + (t["dtype"].is_string() ? t["dtype"].get() : std::string("of no named dtype")) + + "; every tensor of a vision component is BF16 and this reader converts nothing else"); + Tensor x; + std::vector off; + try { + x.shape = t["shape"].get(); + off = t["data_offsets"].get>(); + } catch (const json::exception&) { + fail(name + " has a shape or data_offsets that is not a list of integers"); + } + if (off.size() != 2 || off[1] < off[0] || off[1] > payload_) + fail(name + " has data_offsets outside the payload (" + std::to_string(payload_) + " B)"); + uint64_t n = 0; + if (!elements_of(x.shape, &n)) fail(name + " has shape " + shape_text(x.shape) + ", which is not a size"); + x.bytes = off[1] - off[0]; + if (x.bytes != n * 2) + fail(name + " spans " + std::to_string(x.bytes) + " B but its shape " + shape_text(x.shape) + " is " + + std::to_string(n * 2) + " B of BF16"); + x.offset = off[0]; + x.data = base + off[0]; + tensors_.emplace(name, std::move(x)); + } + if (tensors_.empty()) fail("holds no tensors"); + + // ── no two tensors share a byte ── + { + std::vector> order; + for (const auto& [name, t] : tensors_) order.push_back({t.offset, &name}); + std::sort(order.begin(), order.end()); + uint64_t end = 0, covered = 0; + const std::string* prev = nullptr; + for (const auto& [off, name] : order) { + if (prev && off < end) fail(*name + " overlaps " + *prev + " in the payload"); + end = off + tensors_.at(*name).bytes; + covered += tensors_.at(*name).bytes; + prev = name; + } + unaccounted_ = payload_ - covered; + } + + // ── the geometry, from the shapes ── + const auto need = [&](const std::string& suffix) -> const Tensor& { + const auto it = tensors_.find(kPrefix + suffix); + if (it == tensors_.end()) fail("has no tensor " + std::string(kPrefix) + suffix); + return it->second; + }; + const Shape& pe = need("patch_embed.proj.weight").shape; + if (pe.size() != 4 || pe[2] != pe[3]) + fail("model.visual.patch_embed.proj.weight is " + shape_text(pe) + ", not [hidden, channels, patch, patch]"); + geom_.hidden = as_int(pe[0]); + geom_.channels = as_int(pe[1]); + geom_.patch = as_int(pe[2]); + const int64_t H = geom_.hidden; + + int max_block = -1; + std::vector seen; + const std::string bp = std::string(kPrefix) + "blocks."; + for (const auto& [name, t] : tensors_) { + if (name.compare(0, bp.size(), bp) != 0) continue; + const size_t dot = name.find('.', bp.size()); + const std::string idx = name.substr(bp.size(), dot == std::string::npos ? std::string::npos : dot - bp.size()); + if (dot == std::string::npos || idx.empty() || idx.size() > 4 || idx.find_first_not_of("0123456789") != std::string::npos) + fail(name + " does not name a block index"); + const int i = std::stoi(idx); + if (i >= int(seen.size())) seen.resize(size_t(i) + 1, 0); + seen[size_t(i)] = 1; + max_block = std::max(max_block, i); + } + if (max_block < 0) fail("has no model.visual.blocks.* tensors"); + for (int i = 0; i <= max_block; ++i) + if (!seen[size_t(i)]) fail("has block " + std::to_string(max_block) + " but no block " + std::to_string(i)); + geom_.layers = max_block + 1; + + const Shape& fc1 = need("blocks.0.mlp.linear_fc1.weight").shape; + if (fc1.size() != 2) fail("model.visual.blocks.0.mlp.linear_fc1.weight is " + shape_text(fc1) + ", not a matrix"); + geom_.intermediate = as_int(fc1[0]); + const int64_t I = geom_.intermediate; + + const Shape& pos = need("pos_embed.weight").shape; + if (pos.size() != 2) fail("model.visual.pos_embed.weight is " + shape_text(pos) + ", not [positions, hidden]"); + geom_.pos_rows = as_int(pos[0]); + for (int g = 1; int64_t(g) * g <= geom_.pos_rows; ++g) + if (int64_t(g) * g == geom_.pos_rows) geom_.grid = g; + if (geom_.grid == 0) + fail("model.visual.pos_embed.weight has " + std::to_string(pos[0]) + " rows, which is not a square patch grid"); + + geom_.merge_hidden = int(4 * H); + const int64_t M = geom_.merge_hidden; + const Shape& vfc1 = need("vit_merger.mlp.linear_fc1.weight").shape; + if (vfc1.size() != 2) fail("model.visual.vit_merger.mlp.linear_fc1.weight is " + shape_text(vfc1) + ", not a matrix"); + geom_.vit_merger_intermediate = as_int(vfc1[0]); + const int64_t VI = geom_.vit_merger_intermediate; + const Shape& mfc2 = need("merger.linear_fc2.weight").shape; + if (mfc2.size() != 2) fail("model.visual.merger.linear_fc2.weight is " + shape_text(mfc2) + ", not a matrix"); + geom_.out_hidden = as_int(mfc2[0]); + const int64_t O = geom_.out_hidden; + if (!geom_.hidden || !geom_.channels || !geom_.patch || !geom_.intermediate || !geom_.vit_merger_intermediate || !geom_.out_hidden) + fail("a dimension of the tower is zero or too large to be one"); + + // ── every tensor the tower has, with the shape that geometry gives it; and nothing else in the file ── + std::map want; + const auto add = [&](const std::string& suffix, Shape s) { want[kPrefix + suffix] = std::move(s); }; + const auto linear = [&](const std::string& stem, int64_t rows, int64_t cols) { + add(stem + ".weight", {rows, cols}); + add(stem + ".bias", {rows}); + }; + const auto norm = [&](const std::string& stem, int64_t n) { + add(stem + ".weight", {n}); + add(stem + ".bias", {n}); + }; + add("patch_embed.proj.weight", {H, geom_.channels, geom_.patch, geom_.patch}); + add("patch_embed.proj.bias", {H}); + add("pos_embed.weight", {geom_.pos_rows, H}); + for (int i = 0; i < geom_.layers; ++i) { + const std::string b = "blocks." + std::to_string(i) + "."; + norm(b + "norm1", H); + linear(b + "attn.qkv", 3 * H, H); + linear(b + "attn.proj", H, H); + norm(b + "norm2", H); + linear(b + "mlp.linear_fc1", I, H); + linear(b + "mlp.linear_fc2", H, I); + } + norm("vit_merger.norm1", H); + linear("vit_merger.attn.qkv", 3 * H, H); + linear("vit_merger.attn.proj", H, H); + norm("vit_merger.pre_norm", M); + linear("vit_merger.mlp.linear_fc1", VI, M); + linear("vit_merger.mlp.linear_fc2", H, VI); + norm("merger.norm", H); + norm("merger.pre_norm", M); + linear("merger.linear_fc1", M, M); + linear("merger.linear_fc2", O, M); + + for (const auto& [name, shape] : want) { + const auto it = tensors_.find(name); + if (it == tensors_.end()) fail("has no tensor " + name); + if (it->second.shape != shape) + fail(name + " is " + shape_text(it->second.shape) + " where the tower's other tensors make it " + shape_text(shape)); + } + for (const auto& [name, t] : tensors_) + if (!want.count(name)) fail("holds " + name + " " + shape_text(t.shape) + ", a tensor this reader has no place for"); + + // ── config.json: what the tensors cannot say, and what ties the tower to something outside it ── + geom_.model_intermediate = geom_.intermediate; + geom_.model_vit_merger_intermediate = geom_.vit_merger_intermediate; + if (config_json.empty()) return; + const json c = json::parse(config_json, nullptr, false); + if (c.is_discarded() || !c.is_object()) fail("the config beside it is not a JSON object"); + const json vc = c.contains("vision_config") && c["vision_config"].is_object() ? c["vision_config"] : json::object(); + const auto field = [&](const char* key, int* out) { + const std::string k = std::string("MINICPM_V_") + key; + if (!vc.contains(k) || !vc[k].is_number_integer()) return false; + *out = as_int(vc[k].get()); + return true; + }; + const auto refuse = [&](const char* key, int stated, const std::string& held) { + fail("config.json states MINICPM_V_" + std::string(key) + " " + std::to_string(stated) + " but " + held); + }; + int v = 0; + if (field("VISION_EMBED_DIM", &v) && v != geom_.hidden) + refuse("VISION_EMBED_DIM", v, "the patch embedding is " + std::to_string(geom_.hidden) + " wide"); + if (field("VISION_NUM_LAYERS", &v) && v != geom_.layers) + refuse("VISION_NUM_LAYERS", v, "the container holds " + std::to_string(geom_.layers) + " blocks"); + if (field("PATCH_SIZE", &v) && v != geom_.patch) + refuse("PATCH_SIZE", v, "the patch embedding's kernel is " + std::to_string(geom_.patch)); + if (field("VISION_NUM_POSITION_EMBEDDINGS", &v) && v != geom_.pos_rows) + refuse("VISION_NUM_POSITION_EMBEDDINGS", v, "the position table has " + std::to_string(geom_.pos_rows) + " rows"); + if (field("VISION_OUT_HIDDEN_SIZE", &v) && v != geom_.out_hidden) + refuse("VISION_OUT_HIDDEN_SIZE", v, "the merger projects to " + std::to_string(geom_.out_hidden)); + if (field("MERGER_HIDDEN_SIZE", &v) && v != geom_.merge_hidden) + refuse("MERGER_HIDDEN_SIZE", v, "four tokens of this tower are " + std::to_string(geom_.merge_hidden) + " wide"); + if (field("IMAGE_SIZE", &v)) { + geom_.image_size = v; + if (v != geom_.grid * geom_.patch) + refuse("IMAGE_SIZE", v, "the position table is a " + std::to_string(geom_.grid) + " x " + std::to_string(geom_.grid) + + " grid of " + std::to_string(geom_.patch) + "-pixel patches (" + + std::to_string(geom_.grid * geom_.patch) + " pixels)"); + } + int heads = 0, head_dim = 0; + const bool have_heads = field("VISION_NUM_HEADS", &heads), have_dim = field("VISION_HEAD_DIM", &head_dim); + if (have_heads && !have_dim && heads > 0 && geom_.hidden % heads == 0) head_dim = geom_.hidden / heads; + if (have_heads) { + if (heads <= 0 || int64_t(heads) * head_dim != H) + refuse("VISION_NUM_HEADS", heads, "head_dim " + std::to_string(head_dim) + " times that is not the tower's width " + + std::to_string(geom_.hidden)); + geom_.heads = heads; + geom_.head_dim = head_dim; + } + if (c.contains("hidden_size") && c["hidden_size"].is_number_integer() && c["hidden_size"].get() != O) + fail("config.json states the text model's hidden_size " + std::to_string(c["hidden_size"].get()) + + " but the merger projects to " + std::to_string(geom_.out_hidden) + ": this tower is not this text model's"); + + // An MLP's stated width against its tensors. `stems` are the MLPs that share the width (every block, or the + // vit_merger alone): fc1 is [width, in] with a [width] bias, fc2 is [out, width]. + const auto zero_from = [&](const std::string& name, int64_t rows, int64_t cols, int64_t from, bool by_row) { + const Tensor& t = tensors_.at(kPrefix + name); + const auto is_zero = [&](int64_t i) { + uint16_t h; + std::memcpy(&h, t.data + 2 * i, 2); + return (h & 0x7fff) == 0; + }; + if (by_row) { + for (int64_t i = from * cols; i < rows * cols; ++i) + if (!is_zero(i)) return false; + } else { + for (int64_t r = 0; r < rows; ++r) + for (int64_t c = from; c < cols; ++c) + if (!is_zero(r * cols + c)) return false; + } + return true; + }; + const auto mlp_width = [&](const char* key, int held, const std::vector& stems, int64_t in, int64_t out, + const char* tensor, int* model) { + int stated = 0; + *model = held; + if (!field(key, &stated) || stated == held) return; + if (stated > held) + refuse(key, stated, std::string(tensor) + " has only " + std::to_string(held) + " rows"); + bool pad = true; + for (const std::string& st : stems) + pad = pad && zero_from(st + ".linear_fc1.weight", held, in, stated, true) && + zero_from(st + ".linear_fc1.bias", held, 1, stated, true) && + zero_from(st + ".linear_fc2.weight", out, held, stated, false); + const std::string head = "config.json states MINICPM_V_" + std::string(key) + " " + std::to_string(stated) + "; " + tensor + + " has " + std::to_string(held) + " rows"; + if (pad) { + *model = stated; + notes_.push_back(head + ", the last " + std::to_string(held - stated) + + " all zero: the container pads the model's width, and sizes come from the tensor"); + } else { + notes_.push_back(head + " and the rows past the stated width are not zero: this is not padding, the tensor's " + "width is the model's"); + } + }; + std::vector block_mlps; + for (int i = 0; i < geom_.layers; ++i) block_mlps.push_back("blocks." + std::to_string(i) + ".mlp"); + mlp_width("VISION_MLP_INTERMEDIATE_SIZE", geom_.intermediate, block_mlps, H, H, + "model.visual.blocks.*.mlp.linear_fc1.weight", &geom_.model_intermediate); + mlp_width("VIT_MERGER_INTERMEDIATE_SIZE", geom_.vit_merger_intermediate, {"vit_merger.mlp"}, M, H, + "model.visual.vit_merger.mlp.linear_fc1.weight", &geom_.model_vit_merger_intermediate); +} + +const Tensor& Pack::tensor(const std::string& name) const { + const auto it = tensors_.find(name); + if (it == tensors_.end()) throw std::runtime_error(path_ + ": has no tensor " + name); + return it->second; +} + +std::vector Pack::f32(const std::string& name) const { + const Tensor& t = tensor(name); + std::vector out(static_cast(t.elements())); + for (size_t i = 0; i < out.size(); ++i) { + uint16_t h = 0; + std::memcpy(&h, t.data + 2 * i, 2); + const uint32_t bits = uint32_t(h) << 16; + std::memcpy(&out[i], &bits, 4); + } + return out; +} + +std::string Pack::metadata(const std::string& key) const { + const auto it = meta_.find(key); + return it == meta_.end() ? std::string() : it->second; +} + +int print_info(const std::string& pack_dir) { + const Pack p(pack_dir); + const Geometry& g = p.geometry(); + std::printf("vision component %s\n", p.path().c_str()); + std::printf("tensors %zu, all BF16, %llu B of payload, %llu B unaccounted\n", p.tensors().size(), + (unsigned long long)p.payload_bytes(), (unsigned long long)p.unaccounted_bytes()); + std::printf("patch embed %d x %d pixels x %d channels -> %d\n", g.patch, g.patch, g.channels, g.hidden); + std::printf("positions %d (a %d x %d grid; image %d)\n", g.pos_rows, g.grid, g.grid, g.grid * g.patch); + std::printf("blocks %d at width %d, %d heads x %d, MLP %d (%d stored)\n", g.layers, g.hidden, g.heads, g.head_dim, + g.model_intermediate, g.intermediate); + std::printf("vit_merger attention at %d, MLP %d -> %d (%d stored) -> %d\n", g.hidden, g.merge_hidden, + g.model_vit_merger_intermediate, g.vit_merger_intermediate, g.hidden); + std::printf("merger %d -> %d -> %d (the text model's hidden size)\n", g.merge_hidden, g.merge_hidden, g.out_hidden); + for (const char* k : {"source_file", "source_sha256", "source_dtype", "converter"}) + if (!p.metadata(k).empty()) std::printf("%-17s %s\n", k, p.metadata(k).c_str()); + for (const std::string& n : p.notes()) std::printf("note %s\n", n.c_str()); + return 0; +} + +} // namespace onebit::npu::vision diff --git a/npu/vision_pack.h b/npu/vision_pack.h new file mode 100644 index 000000000..951c16d5f --- /dev/null +++ b/npu/vision_pack.h @@ -0,0 +1,115 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// The vision component of a two-component pack (engine#1788): `vision_weight.q4nx` beside `model.q4nx`. +// +// FastFlowLM's MiniCPM-V 4.7 packs ship the vision tower as its own container at the pack root. It is the same +// container as model.q4nx (8-byte little-endian length, a JSON table keyed by tensor name plus `__metadata__`, then +// the payload that `data_offsets` index), holding a different kind of content: a ViT, every tensor BF16, no tiles. +// +// What this reader is for: the forward that will run the tower must never size a buffer from config.json. Both packs +// on the catalogue state `MINICPM_V_VISION_MLP_INTERMEDIATE_SIZE 4304` and `MINICPM_V_VIT_MERGER_INTERMEDIATE_SIZE +// 17216` while their tensors are 4608 and 17280 rows. The config is the MODEL's width (the source GGUF's ffn tensors +// are 4304 and 17216 wide); the container stores each MLP padded with all-zero rows to a multiple of 128, measured +// bit for bit against both packs' sources. A loader that sized a buffer from the config would read 304 rows short of +// what the file holds and misplace everything after it. So the geometry here is DERIVED from the tensors' own shapes, +// every tensor in the file has to be one this reader can place (zero gaps), and the config is only consulted for what +// the tensors cannot say (the head count, the image size, the unpadded MLP widths) and for the widths that tie the +// tower to something outside it (the patch grid and the text model's hidden size). +// +// Two kinds of disagreement with config.json, on purpose: +// refused the head count does not divide the width, the patch grid is not the position table, the tower's output +// is not the text model's hidden size, the width / block count differ, or an MLP is stated WIDER than its +// tensors: the files do not belong together or the container is cut short, and no forward can be right. +// noted an MLP is stated narrower than its tensors. The tensor decides every size. The rows past the stated +// width are checked: all zero in every block means padding (the published packs), and `model_*` below +// carry the stated width; anything else means the config is simply wrong and `model_*` stay the tensors'. +#pragma once + +#include +#include +#include +#include + +namespace onebit::npu::vision { + +struct Tensor { + std::vector shape; + const uint8_t* data = nullptr; // BF16, little-endian, row-major + uint64_t bytes = 0; + uint64_t offset = 0; // from the payload start + uint64_t elements() const { return bytes / 2; } +}; + +// Everything below is read from tensor shapes, except heads / head_dim / image_size (config.json, 0 when no config +// was given). +struct Geometry { + int hidden = 0; // patch_embed.proj.weight rows + int channels = 0, patch = 0; // patch_embed.proj.weight [hidden, channels, patch, patch] + int layers = 0; // blocks 0..layers-1, all present + int intermediate = 0; // blocks.*.mlp.linear_fc1 rows, the same in every block + int pos_rows = 0, grid = 0; // pos_embed.weight rows, and its square root (patches per side) + int merge_hidden = 0; // 4 * hidden: one 2x2 group of tokens, the width both mergers work at + int vit_merger_intermediate = 0; // vit_merger.mlp.linear_fc1 rows + int out_hidden = 0; // merger.linear_fc2 rows: the text model's hidden size + int heads = 0, head_dim = 0, image_size = 0; + // The MLP widths without the container's zero padding: config.json's value when it is smaller than the tensors' + // and every row past it is zero in every block, else the tensors' own. A forward may stop at these; it must + // still index the tensors at `intermediate` / `vit_merger_intermediate`. + int model_intermediate = 0, model_vit_merger_intermediate = 0; +}; + +class Pack { +public: + // `pack_dir` holds config.json and the container config.json names (`vision_model_weight`, default + // vision_weight.q4nx). Throws std::runtime_error naming the file and the tensor or field on every refusal. + explicit Pack(const std::string& pack_dir); + // The container alone, with config.json's text ("" for none: nothing is cross-checked and heads stay 0). + Pack(const std::string& container_path, const std::string& config_json); + ~Pack(); + Pack(const Pack&) = delete; + Pack& operator=(const Pack&) = delete; + + const Geometry& geometry() const { return geom_; } + const std::map& tensors() const { return tensors_; } + const Tensor& tensor(const std::string& name) const; // throws when absent + // The tensor as float32 (a BF16 value is the top half of a float32, so this is exact). + std::vector f32(const std::string& name) const; + // Config fields the tensors overruled, one sentence each; empty when the config agrees or was not given. + const std::vector& notes() const { return notes_; } + // `__metadata__[key]` as text, "" when absent. + std::string metadata(const std::string& key) const; + uint64_t payload_bytes() const { return payload_; } + // Payload bytes no tensor covers. 0 on both published packs. + uint64_t unaccounted_bytes() const { return unaccounted_; } + const std::string& path() const { return path_; } + +private: + void open(const std::string& container_path, const std::string& config_json); + const uint8_t* map_ = nullptr; + size_t size_ = 0; + std::string path_; + std::map tensors_; + std::map meta_; + std::vector notes_; + Geometry geom_; + uint64_t payload_ = 0, unaccounted_ = 0; +}; + +// `1bit npu-pack vision `: the geometry, the tensor count, the provenance and the notes, on stdout. +// Returns 0, or throws what Pack throws. +int print_info(const std::string& pack_dir); + +} // namespace onebit::npu::vision diff --git a/tests/npu_vision_pack_test.cpp b/tests/npu_vision_pack_test.cpp new file mode 100644 index 000000000..9d2fbeb5c --- /dev/null +++ b/tests/npu_vision_pack_test.cpp @@ -0,0 +1,455 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// npu_vision_pack_test +// +// The vision component reader (npu/vision_pack.h, engine#1788). The container is written here, small, with the same +// tensor names and the same shape relations as FastFlowLM's MiniCPM-V 4.7 `vision_weight.q4nx`, so each arm is one +// edit to a container that otherwise loads: +// - it loads, and the geometry is the one the tensors hold; +// - an MLP the container pads with zero rows (the published packs: the model is 4304 wide, 4608 are stored) is sized +// from the tensor, with the model's width carried beside it and the padding verified to be zero; +// - every way the file can disagree with itself, or with the config in a way no forward survives, is refused with +// the tensor or field named. +// When ONEBIT_MINICPM_PACK names a published pack directory, its own numbers are checked too. +#include "../npu/vision_pack.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using nlohmann::json; +namespace vis = onebit::npu::vision; + +int g_fail = 0, g_arms = 0; +void check(bool ok, const std::string& what) { + ++g_arms; + if (!ok) { + ++g_fail; + std::printf("FAIL %s\n", what.c_str()); + } +} + +// The small tower: width 8 (2 heads x 4), 3 channels, 2-pixel patches, a 4 x 4 grid (image 8), 2 blocks with MLP 32, +// vit_merger MLP 64, projecting to a text model 6 wide. +struct Small { + int H = 8, C = 3, P = 2, pos = 16, L = 2, I = 32, VI = 64, O = 6; +}; + +uint16_t bf16(float v) { + uint32_t b; + std::memcpy(&b, &v, 4); + return uint16_t(b >> 16); +} + +// name -> shape, in the order the payload is laid out +std::vector>> layout(const Small& s) { + std::vector>> v; + const std::string p = "model.visual."; + const auto lin = [&](const std::string& n, int64_t r, int64_t c) { + v.push_back({p + n + ".bias", {r}}); + v.push_back({p + n + ".weight", {r, c}}); + }; + const auto nrm = [&](const std::string& n, int64_t k) { + v.push_back({p + n + ".bias", {k}}); + v.push_back({p + n + ".weight", {k}}); + }; + const int64_t H = s.H, M = 4 * s.H; + for (int i = 0; i < s.L; ++i) { + const std::string b = "blocks." + std::to_string(i) + "."; + lin(b + "attn.proj", H, H); + lin(b + "attn.qkv", 3 * H, H); + lin(b + "mlp.linear_fc1", s.I, H); + lin(b + "mlp.linear_fc2", H, s.I); + nrm(b + "norm1", H); + nrm(b + "norm2", H); + } + lin("merger.linear_fc1", M, M); + lin("merger.linear_fc2", s.O, M); + nrm("merger.norm", H); + nrm("merger.pre_norm", M); + v.push_back({p + "patch_embed.proj.bias", {H}}); + v.push_back({p + "patch_embed.proj.weight", {H, s.C, s.P, s.P}}); + v.push_back({p + "pos_embed.weight", {s.pos, H}}); + lin("vit_merger.attn.proj", H, H); + lin("vit_merger.attn.qkv", 3 * H, H); + lin("vit_merger.mlp.linear_fc1", s.VI, M); + lin("vit_merger.mlp.linear_fc2", H, s.VI); + nrm("vit_merger.norm1", H); + nrm("vit_merger.pre_norm", M); + return v; +} + +json config_of(const Small& s) { + json vc; + vc["MINICPM_V_PATCH_SIZE"] = s.P; + vc["MINICPM_V_IMAGE_SIZE"] = 8; + vc["MINICPM_V_VISION_EMBED_DIM"] = s.H; + vc["MINICPM_V_VISION_NUM_HEADS"] = 2; + vc["MINICPM_V_VISION_HEAD_DIM"] = 4; + vc["MINICPM_V_VISION_MLP_INTERMEDIATE_SIZE"] = s.I; + vc["MINICPM_V_VISION_NUM_POSITION_EMBEDDINGS"] = s.pos; + vc["MINICPM_V_VISION_NUM_LAYERS"] = s.L; + vc["MINICPM_V_VISION_OUT_HIDDEN_SIZE"] = s.O; + vc["MINICPM_V_MERGER_HIDDEN_SIZE"] = 4 * s.H; + vc["MINICPM_V_VIT_MERGER_INTERMEDIATE_SIZE"] = s.VI; + json c; + c["hidden_size"] = s.O; + c["vision_model_weight"] = "vision_weight.q4nx"; + c["vision_config"] = vc; + return c; +} + +// Writes the container. `edit` sees the table (and may change the payload size) before it is serialised. Element e of +// the tensor at payload offset `off` holds bf16(0.25 * ((off/2 + e) % 64) - 4): readable back by position. +void write(const std::string& path, const Small& s, const std::function&)>& edit = {}) { + json table = json::object(); + std::vector payload; + for (const auto& [name, shape] : layout(s)) { + uint64_t n = 1; + for (int64_t d : shape) n *= uint64_t(d); + const uint64_t off = payload.size(); + for (uint64_t e = 0; e < n; ++e) { + const uint16_t h = bf16(0.25f * float((off / 2 + e) % 64) - 4.0f); + payload.push_back(uint8_t(h & 0xff)); + payload.push_back(uint8_t(h >> 8)); + } + table[name] = {{"dtype", "BF16"}, {"shape", shape}, {"data_offsets", {off, off + n * 2}}}; + } + table["__metadata__"] = {{"source_file", "crafted.gguf"}, {"converter", "npu_vision_pack_test"}}; + if (edit) edit(table, payload); + const std::string head = table.dump(); + const uint64_t len = head.size(); + std::ofstream f(path, std::ios::binary | std::ios::trunc); + f.write(reinterpret_cast(&len), 8); + f.write(head.data(), std::streamsize(head.size())); + f.write(reinterpret_cast(payload.data()), std::streamsize(payload.size())); +} + +const char* const kPath = "npu_vision_pack_test.q4nx"; + +// Loads the crafted file; "" when it loads, else what it threw. +std::string load(const std::string& config) { + try { + const vis::Pack p(kPath, config); + return ""; + } catch (const std::exception& e) { + return e.what(); + } +} + +void refused(const std::string& arm, const std::string& config, const std::vector& must_name) { + const std::string why = load(config); + check(!why.empty(), arm + ": refused"); + for (const std::string& m : must_name) + check(why.find(m) != std::string::npos, arm + ": the refusal names \"" + m + "\" (it says: " + why + ")"); +} + +void test_loads() { + const Small s; + write(kPath, s); + const std::string cfg = config_of(s).dump(); + try { + const vis::Pack p(kPath, cfg); + const vis::Geometry& g = p.geometry(); + check(g.hidden == 8 && g.channels == 3 && g.patch == 2, "geometry: the patch embedding's own shape"); + check(g.layers == 2 && g.intermediate == 32, "geometry: two blocks, MLP 32"); + check(g.pos_rows == 16 && g.grid == 4, "geometry: 16 positions are a 4 x 4 grid"); + check(g.merge_hidden == 32 && g.vit_merger_intermediate == 64 && g.out_hidden == 6, "geometry: the two mergers"); + check(g.heads == 2 && g.head_dim == 4 && g.image_size == 8, "geometry: heads and image size come from the config"); + check(p.tensors().size() == size_t(2 * 12 + 23), "every tensor is placed: 12 per block and 23 outside them"); + check(p.unaccounted_bytes() == 0, "no payload byte is outside a tensor"); + check(p.notes().empty(), "a config that agrees leaves no note"); + check(p.metadata("source_file") == "crafted.gguf" && p.metadata("absent").empty(), "metadata is read by key"); + // exact values back, by position: the first tensor starts at payload offset 0 + const std::vector b = p.f32("model.visual.blocks.0.attn.proj.bias"); + bool ok = b.size() == 8; + for (size_t e = 0; ok && e < b.size(); ++e) ok = b[e] == 0.25f * float(e % 64) - 4.0f; + check(ok, "f32(): BF16 values come back exactly"); + const vis::Tensor& t = p.tensor("model.visual.pos_embed.weight"); + const std::vector pos = p.f32("model.visual.pos_embed.weight"); + check(pos.size() == 16 * 8 && pos[3] == 0.25f * float((t.offset / 2 + 3) % 64) - 4.0f, + "f32(): a tensor in the middle of the payload reads from its own offset"); + } catch (const std::exception& e) { + check(false, std::string("the crafted container loads (threw: ") + e.what() + ")"); + } + // with no config nothing is cross-checked and the config-only fields stay 0 + try { + const vis::Pack p(kPath, ""); + check(p.geometry().hidden == 8 && p.geometry().heads == 0 && p.geometry().image_size == 0, + "no config: the tensor geometry stands, heads and image size are unknown"); + } catch (const std::exception& e) { + check(false, std::string("loads without a config (threw: ") + e.what() + ")"); + } +} + +// Zeroes the rows of an MLP past `from`: fc1 weight rows and bias entries, fc2 weight columns. This is what the +// published packs hold past the model's width. +void zero_mlp_tail(json& t, std::vector& p, const std::string& stem, int from) { + const auto span = [&](const std::string& n, uint64_t* off, std::vector* shape) { + *off = t[n]["data_offsets"][0]; + *shape = t[n]["shape"].get>(); + }; + uint64_t off; + std::vector sh; + span(stem + ".linear_fc1.weight", &off, &sh); + for (int64_t i = int64_t(from) * sh[1]; i < sh[0] * sh[1]; ++i) p[off + 2 * i] = p[off + 2 * i + 1] = 0; + span(stem + ".linear_fc1.bias", &off, &sh); + for (int64_t i = from; i < sh[0]; ++i) p[off + 2 * i] = p[off + 2 * i + 1] = 0; + span(stem + ".linear_fc2.weight", &off, &sh); + for (int64_t r = 0; r < sh[0]; ++r) + for (int64_t c = from; c < sh[1]; ++c) p[off + 2 * (r * sh[1] + c)] = p[off + 2 * (r * sh[1] + c) + 1] = 0; +} + +// The published packs' situation: the config states the model's MLP widths (4304, 17216) and the container stores them +// padded with zero rows (4608, 17280). Every size is the tensor's; the model's width is carried beside it. +void test_padded_mlp_widths() { + const Small s; + json c = config_of(s); + c["vision_config"]["MINICPM_V_VISION_MLP_INTERMEDIATE_SIZE"] = 30; + c["vision_config"]["MINICPM_V_VIT_MERGER_INTERMEDIATE_SIZE"] = 60; + const auto pad_all = [&](json& t, std::vector& p) { + zero_mlp_tail(t, p, "model.visual.blocks.0.mlp", 30); + zero_mlp_tail(t, p, "model.visual.blocks.1.mlp", 30); + zero_mlp_tail(t, p, "model.visual.vit_merger.mlp", 60); + }; + write(kPath, s, pad_all); + try { + const vis::Pack p(kPath, c.dump()); + const vis::Geometry& g = p.geometry(); + check(g.intermediate == 32 && g.vit_merger_intermediate == 64, "padded MLP: both stored widths are the tensors'"); + check(g.model_intermediate == 30 && g.model_vit_merger_intermediate == 60, + "padded MLP: the model's widths are the config's, because every row past them is zero"); + check(p.notes().size() == 2, "padded MLP: one note per padded width"); + const std::string all = p.notes().size() == 2 ? p.notes()[0] + p.notes()[1] : ""; + check(all.find("VISION_MLP_INTERMEDIATE_SIZE 30") != std::string::npos && all.find("32 rows, the last 2 all zero") != std::string::npos, + "padded MLP: the note states the config's value, the tensor's, and that the difference is zero rows"); + check(all.find("VIT_MERGER_INTERMEDIATE_SIZE 60") != std::string::npos && all.find("64 rows, the last 4 all zero") != std::string::npos, + "padded MLP: and the same for the vit_merger"); + } catch (const std::exception& e) { + check(false, std::string("a container that pads its MLPs loads (threw: ") + e.what() + ")"); + } + // one nonzero value past the stated width, in ONE block's fc2: that is not padding, and it is said so + write(kPath, s, [&](json& t, std::vector& p) { + pad_all(t, p); + const uint64_t off = t["model.visual.blocks.1.mlp.linear_fc2.weight"]["data_offsets"][0]; + const uint64_t e = uint64_t(7) * 32 + 31; // last row, last column + p[off + 2 * e + 1] = 0x3f; // bf16 high byte: a nonzero value + }); + try { + const vis::Pack p(kPath, c.dump()); + check(p.geometry().model_intermediate == 32, "not padding: one nonzero value past the stated width keeps the tensor's width"); + check(p.geometry().model_vit_merger_intermediate == 60, "not padding: the vit_merger, still zero past 60, keeps the config's"); + const std::string all = p.notes().size() == 2 ? p.notes()[0] + p.notes()[1] : ""; + check(all.find("are not zero: this is not padding") != std::string::npos, "not padding: the note says so"); + } catch (const std::exception& e) { + check(false, std::string("a config narrower than nonzero tensors still loads (threw: ") + e.what() + ")"); + } + // without a config the model's widths are the stored ones + try { + const vis::Pack p(kPath, ""); + check(p.geometry().model_intermediate == 32 && p.geometry().model_vit_merger_intermediate == 64, + "no config: the model's widths are the stored widths"); + } catch (const std::exception& e) { + check(false, std::string("loads without a config (threw: ") + e.what() + ")"); + } + // a config that states an MLP WIDER than its tensors cannot be padding + json wide = config_of(s); + wide["vision_config"]["MINICPM_V_VISION_MLP_INTERMEDIATE_SIZE"] = 40; + write(kPath, s); + refused("config: an MLP wider than its tensors", wide.dump(), {"VISION_MLP_INTERMEDIATE_SIZE 40", "only 32 rows"}); +} + +void test_refusals() { + const Small s; + const std::string cfg = config_of(s).dump(); + const std::string qkv0 = "model.visual.blocks.0.attn.qkv.weight"; + + write(kPath, s, [&](json& t, std::vector&) { t[qkv0]["dtype"] = "F32"; }); + refused("a tensor that is not BF16", cfg, {qkv0, "F32"}); + + write(kPath, s, [&](json& t, std::vector&) { t[qkv0]["shape"] = {24, 9}; }); + refused("a shape that is not the tensor's byte length", cfg, {qkv0, "[24, 9]"}); + + // the shape the tower expects, over one element fewer bytes: a forward would read the last element out of the + // NEXT tensor. Only the extent-against-shape check can see this one. + write(kPath, s, [&](json& t, std::vector&) { + const uint64_t a = t[qkv0]["data_offsets"][0], b = t[qkv0]["data_offsets"][1]; + t[qkv0]["data_offsets"] = {a, b - 2}; + }); + refused("an extent shorter than the shape", cfg, {qkv0, "spans 382 B", "384 B of BF16"}); + + write(kPath, s, [&](json& t, std::vector& p) { + const uint64_t a = t[qkv0]["data_offsets"][0]; + t[qkv0]["data_offsets"] = {a + p.size(), a + p.size() + 384}; + }); + refused("offsets past the payload", cfg, {qkv0, "outside the payload"}); + + write(kPath, s, [&](json& t, std::vector&) { + // norm2.bias moved onto norm1.bias: same length, so only the overlap check can see it + t["model.visual.blocks.0.norm2.bias"]["data_offsets"] = t["model.visual.blocks.0.norm1.bias"]["data_offsets"]; + }); + refused("two tensors on the same bytes", cfg, {"overlaps", "model.visual.blocks.0.norm"}); + + write(kPath, s, [&](json& t, std::vector&) { t.erase("model.visual.blocks.1.norm2.bias"); }); + refused("a missing tensor", cfg, {"has no tensor model.visual.blocks.1.norm2.bias"}); + + write(kPath, s, [&](json& t, std::vector&) { + json moved = json::object(); + for (auto it = t.begin(); it != t.end(); ++it) { + std::string k = it.key(); + const std::string from = "model.visual.blocks.1."; + if (k.compare(0, from.size(), from) == 0) k = "model.visual.blocks.2." + k.substr(from.size()); + moved[k] = it.value(); + } + t = moved; + }); + refused("a hole in the block indices", cfg, {"block 2", "no block 1"}); + + write(kPath, s, [&](json& t, std::vector& p) { + const uint64_t off = p.size(); + p.resize(p.size() + 16, 0); + t["model.visual.blocks.0.attn.gate.weight"] = {{"dtype", "BF16"}, {"shape", {8}}, {"data_offsets", {off, off + 16}}}; + }); + refused("a tensor the tower has no place for", cfg, {"model.visual.blocks.0.attn.gate.weight", "no place"}); + + // one block with a different MLP width: its own extents are consistent, so only the cross-block shape check sees it + write(kPath, s, [&](json& t, std::vector& p) { + const auto grow = [&](const std::string& n, std::vector shape) { + uint64_t k = 2; + for (int64_t d : shape) k *= uint64_t(d); + const uint64_t off = p.size(); + p.resize(p.size() + k, 0); + t[n] = {{"dtype", "BF16"}, {"shape", shape}, {"data_offsets", {off, off + k}}}; + }; + grow("model.visual.blocks.1.mlp.linear_fc1.weight", {40, 8}); + grow("model.visual.blocks.1.mlp.linear_fc1.bias", {40}); + grow("model.visual.blocks.1.mlp.linear_fc2.weight", {8, 40}); + }); + refused("a block whose MLP is another width", "", {"model.visual.blocks.1.mlp.linear_fc1", "[32"}); + + { + Small odd; + odd.pos = 15; + write(kPath, odd); + refused("a position table that is not a square grid", "", {"pos_embed", "15 rows"}); + } + write(kPath, s, [&](json&, std::vector&) {}); + { + std::ofstream f(kPath, std::ios::binary | std::ios::trunc); + const uint64_t len = 1 << 20; + f.write(reinterpret_cast(&len), 8); + f.write("{}", 2); + } + refused("a header length past the end of the file", cfg, {"header runs past the end"}); + + // ── the config against the tensors: refused where no forward survives it ── + write(kPath, s); + const auto with = [&](const char* key, int v) { + json c = config_of(s); + c["vision_config"][std::string("MINICPM_V_") + key] = v; + return c.dump(); + }; + refused("config: heads that do not make the width", with("VISION_NUM_HEADS", 3), {"VISION_NUM_HEADS 3", "width 8"}); + refused("config: another width", with("VISION_EMBED_DIM", 16), {"VISION_EMBED_DIM 16", "8 wide"}); + refused("config: more blocks than the container holds", with("VISION_NUM_LAYERS", 3), {"VISION_NUM_LAYERS 3", "2 blocks"}); + refused("config: another patch size", with("PATCH_SIZE", 4), {"PATCH_SIZE 4"}); + refused("config: an image the position table does not cover", with("IMAGE_SIZE", 12), {"IMAGE_SIZE 12", "4 x 4"}); + refused("config: another position count", with("VISION_NUM_POSITION_EMBEDDINGS", 25), {"NUM_POSITION_EMBEDDINGS 25", "16 rows"}); + { + json c = config_of(s); + c["hidden_size"] = 12; + refused("config: a text model of another width", c.dump(), {"hidden_size 12", "projects to 6"}); + } +} + +// Pack(dir): the container name comes from config.json and must stay inside the pack. +void test_pack_dir_name() { + const Small s; + const std::string dir = "npu_vision_pack_test.dir"; + std::error_code ec; + std::filesystem::create_directories(dir, ec); + write(dir + "/tower.q4nx", s); + json c = config_of(s); + c["vision_model_weight"] = "tower.q4nx"; + { std::ofstream f(dir + "/config.json"); f << c.dump(); } + try { + const vis::Pack p(dir); + check(p.geometry().layers == 2 && p.path() == dir + "/tower.q4nx", "Pack(dir): opens the file config.json names"); + } catch (const std::exception& e) { + check(false, std::string("Pack(dir) opens the named container (threw: ") + e.what() + ")"); + } + c["vision_model_weight"] = "../npu_vision_pack_test.q4nx"; + { std::ofstream f(dir + "/config.json"); f << c.dump(); } + std::string why; + try { const vis::Pack p(dir); } catch (const std::exception& e) { why = e.what(); } + check(why.find("is not a file name") != std::string::npos, "Pack(dir): a name with a separator is refused, not followed"); + std::remove((dir + "/tower.q4nx").c_str()); + std::remove((dir + "/config.json").c_str()); + std::filesystem::remove(dir, ec); +} + +// The published pack, when one is on the box: the numbers engine#1788's thread read by hand. +void test_published_pack(const std::string& dir) { + try { + const vis::Pack p(dir); + const vis::Geometry& g = p.geometry(); + check(p.tensors().size() == 347, "published pack: 347 tensors, all placed"); + check(p.unaccounted_bytes() == 0, "published pack: no payload byte outside a tensor"); + check(g.hidden == 1152 && g.heads == 16 && g.head_dim == 72 && g.layers == 27, "published pack: 27 blocks at 1152, 16 x 72"); + check(g.patch == 14 && g.channels == 3 && g.pos_rows == 4900 && g.grid == 70 && g.image_size == 980, + "published pack: 14-pixel patches on a 70 x 70 grid (980)"); + check(g.intermediate == 4608 && g.vit_merger_intermediate == 17280, "published pack: MLPs stored 4608 and 17280 wide"); + check(g.model_intermediate == 4304 && g.model_vit_merger_intermediate == 17216, + "published pack: the model's MLP widths are 4304 and 17216, the rest verified zero in all 27 blocks"); + check(g.merge_hidden == 4608, "published pack: mergers at 4 x 1152"); + check(p.notes().size() == 2, "published pack: both padded widths are noted"); + // The 1B pack's container names its source; the 35B-A3B's carries no __metadata__ at all. Both are accepted, + // and a source hash that is present has to look like one. + const std::string sha = p.metadata("source_sha256"); + check(sha.empty() || (sha.size() == 64 && sha.find_first_not_of("0123456789abcdef") == std::string::npos), + "published pack: a source hash, when the container carries one, is 64 hex digits"); + std::printf(" provenance: %s\n", sha.empty() ? "none in the container" : (p.metadata("source_file") + " sha256 " + sha).c_str()); + std::printf(" published pack %s: %zu tensors, out_hidden %d, %zu note(s)\n", dir.c_str(), p.tensors().size(), g.out_hidden, + p.notes().size()); + for (const std::string& n : p.notes()) std::printf(" note: %s\n", n.c_str()); + } catch (const std::exception& e) { + check(false, std::string("the published pack loads (threw: ") + e.what() + ")"); + } +} + +} // namespace + +int main() { + test_loads(); + test_padded_mlp_widths(); + test_refusals(); + test_pack_dir_name(); + std::remove(kPath); + if (const char* dir = std::getenv("ONEBIT_MINICPM_PACK"); dir && *dir) test_published_pack(dir); + else std::printf(" (ONEBIT_MINICPM_PACK not set: the published-pack arm did not run)\n"); + std::printf("vision pack: %d checks, %d failed\n", g_arms, g_fail); + return g_fail ? 1 : 0; +}