From 85a128dce8db651ef6916bac34d2985edb4ac727 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 19:03:20 -0400 Subject: [PATCH 1/3] npu: the MiniCPM-V 4.7 image plan: sizes, slice grid and text rows, matching the reference processor (engine#1788) vision::plan_image(height, width) is the arithmetic of the model's image processor with no pixel touched: the whole view, the slice grid (up to 9), the resize the slices are cut from, and the rows each piece becomes after the tower's two 2x2 merges. Python's round-half-even and int() truncation are followed exactly. Test: npu_vision_plan, against transformers' MiniCPMV4_6ImageProcessor 5.22.2 for 3,481 image sizes (one hash over every number, 16 plans written out). Five mutants caught; a sixth was equivalent and its claim removed from the header. Co-Authored-By: Claude Fable 5.2 --- CMakeLists.txt | 5 ++ docs/npu.md | 20 +++++ npu/vision_plan.h | 147 +++++++++++++++++++++++++++++++++ tests/npu_vision_plan_test.cpp | 135 ++++++++++++++++++++++++++++++ 4 files changed, 307 insertions(+) create mode 100644 npu/vision_plan.h create mode 100644 tests/npu_vision_plan_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index a90354cea..d0b398b4a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1219,6 +1219,11 @@ add_executable(npu_vision_tower_test tests/npu_vision_tower_test.cpp) target_link_libraries(npu_vision_tower_test PRIVATE onebit_npu) add_test(NAME npu_vision_tower COMMAND npu_vision_tower_test) add_dependencies(onebit npu_vision_tower_test) +# The image plan (sizes, slice grid, text rows) against the reference image processor's answers for 3,481 sizes. +add_executable(npu_vision_plan_test tests/npu_vision_plan_test.cpp) +target_include_directories(npu_vision_plan_test PRIVATE npu) +add_test(NAME npu_vision_plan COMMAND npu_vision_plan_test) +add_dependencies(onebit npu_vision_plan_test) add_executable(npu_q4nx_test tests/npu_q4nx_test.cpp) target_include_directories(npu_q4nx_test PRIVATE app) target_link_libraries(npu_q4nx_test PRIVATE onebit_npu) diff --git a/docs/npu.md b/docs/npu.md index 1c4396014..fff878f09 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1542,6 +1542,26 @@ output for it as goldens, so the arithmetic is checked in CI without a pack; wit `ONEBIT_MINICPM_PACK` and `ONEBIT_MINICPM_ORACLE` set it repeats the comparison above. The reference run itself needs transformers and torch and is not in the tree. +**Where an image goes before the tower (`npu/vision_plan.h`).** `vision::plan_image(height, width)` is +the arithmetic of the model's image processor (transformers' `MiniCPMV4_6ImageProcessor`, which 4.7 +loads), with no pixel touched: the model looks at an image whole, resized to about 448 × 448 pixels, +and, when the image is larger than that, again as a grid of up to 9 slices cut from a second, larger +resize. Every piece is a multiple of 56 pixels (4 patches) in both extents, because the tower merges +2 × 2 twice, and becomes `(height/56) × (width/56)` rows on the text side. + +| image | whole view | slices | text rows | +|---|---|---|---| +| 448 × 448 or smaller (scaled up) | 448 × 448 | none | 64 | +| 480 × 640 | 392 × 504 | 1 × 2 of 560 × 392 | 63 + 2 × 70 | +| 1080 × 1920 | 336 × 616 | 2 × 4 of 448 × 448 | 66 + 8 × 64 | +| 3024 × 4032 | 392 × 504 | 3 × 3 of 392 × 504 | 63 + 9 × 63 | + +(heights first.) So one 448-pixel slice is 64 rows, not the 1,225 that 980 / 14 / 2 squared suggests: the +position table is 70 × 70, but no piece the processor produces is that large unless the image is a very +long strip. The plan is checked against the reference's own answers for 3,481 image sizes in +`tests/npu_vision_plan_test.cpp`. Resampling the pixels (the reference uses bicubic), rescaling and the +slice separators on the text side are not written. + ## Serving The 1bit NPU engine runs the model on the XDNA 2 NPU. With a release package that carries the NPU diff --git a/npu/vision_plan.h b/npu/vision_plan.h new file mode 100644 index 000000000..9842b3838 --- /dev/null +++ b/npu/vision_plan.h @@ -0,0 +1,147 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Where a MiniCPM-V 4.7 image goes before the vision tower sees it (engine#1788): the sizes, the slice grid, and how +// many rows each piece becomes on the text side. Arithmetic only; no pixel is touched here. +// +// The model looks at an image twice: once whole, resized to about `scale_resolution`^2 pixels, and, when the image is +// larger than that, again as a rows x cols grid of slices cut from a second, larger resize. Every piece is a multiple +// of 4 patches in both extents, because the tower merges 2 x 2 twice (npu/vision_tower.h), and becomes +// (height/patch/4) * (width/patch/4) rows. +// +// This is transformers' MiniCPMV4_6ImageProcessor (the image processor MiniCPM-V 4.7 loads), followed step by step, +// including the places where the obvious arithmetic gives a different answer: +// - `round()` there is Python's: halves go to the EVEN neighbour (2.5 -> 2), which std::round does not do; +// - `int()` truncates, and the width is computed from the already truncated height; +// - the slice grid tries one piece fewer and one more than area / scale^2, and a tie goes to more rows; +// - an image that is not sliced is scaled UP to the resolution; one that is sliced has its whole view scaled down only. +// (The per-slice size, refine / grid, is an exact division: the refine size was just rounded to a multiple of the grid.) +// tests/npu_vision_plan_test.cpp holds that implementation's answers for 3,481 image sizes. +#pragma once + +#include +#include +#include +#include +#include + +namespace onebit::npu::vision { + +struct PiecePlan { + int height = 0, width = 0; // pixels the piece is resized (source) or cut (slice) to + int grid_h = 0, grid_w = 0; // patches + int rows = 0; // rows on the text side: grid_h/4 * grid_w/4 +}; + +struct ImagePlan { + PiecePlan source; // the whole image + int slice_rows = 0, slice_cols = 0; // 0 x 0: the image is not sliced + int refine_height = 0, refine_width = 0; // the resize the slices are cut from (slice_rows x slice_cols equal pieces) + PiecePlan slice; // every slice: they are all the same size + // Everything the tower is run on, in the order the reference feeds it: the source, then the slices row by row. + int pieces() const { return 1 + slice_rows * slice_cols; } + int total_rows() const { return source.rows + slice_rows * slice_cols * slice.rows; } +}; + +namespace plan_detail { + +// Python's round(): to nearest, halves to even. The default floating-point rounding mode is that rule. +inline int64_t py_round(double v) { return int64_t(std::nearbyint(v)); } + +// max(round(length / divisor) * divisor, divisor) +inline int64_t ensure_divide(double length, int64_t divisor) { + const int64_t r = py_round(length / double(divisor)) * divisor; + return r > divisor ? r : divisor; +} + +inline void best_resize(double height, double width, int scale, int patch, bool allow_upscale, int64_t* out_h, int64_t* out_w) { + if (height * width > double(scale) * double(scale) || allow_upscale) { + const double aspect = width / height; + const double h = double(int64_t(double(scale) / std::sqrt(aspect))); // int(): truncation + height = h; + width = double(int64_t(h * aspect)); + } + const int64_t unit = int64_t(patch) * 4; // two 2 x 2 merges + *out_w = ensure_divide(width, unit); + *out_h = ensure_divide(height, unit); +} + +} // namespace plan_detail + +// The plan for a height x width image. The defaults are the model's (preprocessor_config.json: max_slice_nums 9; the +// image processor's scale_resolution 448; the tower's 14-pixel patch). Throws on a size that is not an image or that +// the arithmetic cannot hold. +inline ImagePlan plan_image(int height, int width, int max_slice_nums = 9, int scale_resolution = 448, int patch = 14) { + using namespace plan_detail; + if (height <= 0 || width <= 0 || height > (1 << 20) || width > (1 << 20)) + throw std::runtime_error("an image of " + std::to_string(height) + " x " + std::to_string(width) + " pixels cannot be planned"); + if (max_slice_nums < 1 || max_slice_nums > 64 || scale_resolution < 4 * patch || scale_resolution > 8192 || patch < 1) + throw std::runtime_error("the image plan's parameters are out of range"); + ImagePlan p; + const double H = height, W = width; + + // the slice grid: the factorisation of about (area / scale^2) pieces whose shape is closest to the image's + const double log_ratio = std::log(W / H); + const double ratio = W * H / (double(scale_resolution) * double(scale_resolution)); + const int64_t ceil_ratio = int64_t(std::ceil(ratio)); + const int multiple = int(ceil_ratio < max_slice_nums ? ceil_ratio : max_slice_nums); + if (multiple > 1) { + int best_rows = 1, best_cols = 1; + double min_error = INFINITY; + for (int n = multiple - 1; n <= multiple + 1; ++n) { + if (n == 1 || n > max_slice_nums) continue; + for (int rows = 1; rows <= n; ++rows) { + if (n % rows != 0) continue; + const int cols = n / rows; + const double error = std::fabs(log_ratio - std::log(double(cols) / double(rows))); + if (error < min_error || (error == min_error && rows > best_rows)) { + best_rows = rows; + best_cols = cols; + min_error = error; + } + } + } + p.slice_rows = best_rows; + p.slice_cols = best_cols; + } + const bool sliced = p.slice_rows > 0; + + const auto piece = [&](int64_t h, int64_t w) { + if (h > (int64_t(1) << 24) || w > (int64_t(1) << 24)) throw std::runtime_error("the image plan does not fit"); + PiecePlan q; + q.height = int(h); + q.width = int(w); + q.grid_h = q.height / patch; + q.grid_w = q.width / patch; + q.rows = (q.grid_h / 4) * (q.grid_w / 4); + return q; + }; + int64_t sh = 0, sw = 0; + best_resize(H, W, scale_resolution, patch, !sliced, &sh, &sw); // a small image is scaled UP to the resolution + p.source = piece(sh, sw); + + if (sliced) { + const int64_t refine_w = ensure_divide(W, p.slice_cols), refine_h = ensure_divide(H, p.slice_rows); + int64_t bh = 0, bw = 0; + best_resize(double(refine_h) / double(p.slice_rows), double(refine_w) / double(p.slice_cols), scale_resolution, patch, true, + &bh, &bw); + p.slice = piece(bh, bw); + p.refine_height = p.slice.height * p.slice_rows; + p.refine_width = p.slice.width * p.slice_cols; + } + return p; +} + +} // namespace onebit::npu::vision diff --git a/tests/npu_vision_plan_test.cpp b/tests/npu_vision_plan_test.cpp new file mode 100644 index 000000000..bce00c382 --- /dev/null +++ b/tests/npu_vision_plan_test.cpp @@ -0,0 +1,135 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// npu_vision_plan_test +// +// The image plan (npu/vision_plan.h, engine#1788) against transformers' MiniCPMV4_6ImageProcessor 5.22.2: the methods +// find_best_resize, get_refine_size and get_sliced_grid, called as the processor's _preprocess calls them, for every +// pair of the 59 sizes below (3,481 images). The eight numbers of each plan are folded into one FNV-1a hash, compared +// with the hash the reference produces for the same walk; sixteen plans are also written out so a failure says where. +#include "../npu/vision_plan.h" + +#include +#include +#include +#include + +namespace { + +namespace vis = onebit::npu::vision; + +int g_fail = 0, g_arms = 0; +void check(bool ok, const std::string& what) { + ++g_arms; + if (!ok) { + ++g_fail; + std::printf("FAIL %s\n", what.c_str()); + } +} + +const int kSizes[] = {1, 2, 3, 7, 13, 14, 27, 28, 55, 56, 57, 100, 111, 224, 300, + 336, 392, 419, 420, 447, 448, 449, 476, 500, 503, 504, 600, 640, 672, 700, + 720, 768, 800, 896, 900, 960, 1000, 1024, 1080, 1200, 1280, 1344, 1440, 1536, 1600, + 1792, 1920, 2000, 2048, 2160, 2560, 3000, 3024, 3840, 4032, 4096, 5000, 6000, 8000}; + +// {height, width, {source h, source w, slice rows, slice cols, refine h, refine w, slice h, slice w}}, from the reference +struct Row { + int h, w, plan[8]; +}; +const Row kRows[] = { + {448, 448, {448, 448, 0, 0, 0, 0, 0, 0}}, + {1, 1, {448, 448, 0, 0, 0, 0, 0, 0}}, + {100, 100, {448, 448, 0, 0, 0, 0, 0, 0}}, + {449, 448, {448, 448, 2, 1, 672, 616, 336, 616}}, + {480, 640, {392, 504, 1, 2, 560, 784, 560, 392}}, + {720, 1280, {336, 616, 2, 3, 784, 1512, 392, 504}}, + {1080, 1920, {336, 616, 2, 4, 896, 1792, 448, 448}}, + {1920, 1080, {616, 336, 4, 2, 1792, 896, 448, 448}}, + {3024, 4032, {392, 504, 3, 3, 1176, 1512, 392, 504}}, + {224, 2000, {168, 1344, 1, 4, 280, 2688, 280, 672}}, + {5000, 300, {1848, 112, 9, 1, 5544, 336, 616, 336}}, + {896, 896, {448, 448, 2, 2, 896, 896, 448, 448}}, + {1344, 1344, {448, 448, 3, 3, 1344, 1344, 448, 448}}, + {600, 800, {392, 504, 2, 2, 784, 1008, 392, 504}}, + {8000, 8000, {448, 448, 3, 3, 1344, 1344, 448, 448}}, + {14, 8000, {56, 10304, 0, 0, 0, 0, 0, 0}}, +}; +const uint64_t kReferenceHash = 0x8480087394bf204fULL; // the walk below, by the reference +const int kReferenceMaxRows = 716; // the most text rows any of the 3,481 plans gives + +void eight(const vis::ImagePlan& p, int out[8]) { + out[0] = p.source.height; + out[1] = p.source.width; + out[2] = p.slice_rows; + out[3] = p.slice_cols; + out[4] = p.refine_height; + out[5] = p.refine_width; + out[6] = p.slice.height; + out[7] = p.slice.width; +} + +} // namespace + +int main() { + for (const Row& r : kRows) { + int got[8]; + eight(vis::plan_image(r.h, r.w), got); + bool same = true; + for (int i = 0; i < 8; ++i) same = same && got[i] == r.plan[i]; + check(same, std::to_string(r.h) + " x " + std::to_string(r.w) + ": source " + std::to_string(got[0]) + " x " + + std::to_string(got[1]) + ", grid " + std::to_string(got[2]) + " x " + std::to_string(got[3]) + ", refine " + + std::to_string(got[4]) + " x " + std::to_string(got[5]) + ", slice " + std::to_string(got[6]) + " x " + + std::to_string(got[7]) + " is the reference's plan"); + } + + uint64_t h = 0xcbf29ce484222325ULL; + int max_rows = 0, pairs = 0; + bool shapes_ok = true; + for (int height : kSizes) + for (int width : kSizes) { + const vis::ImagePlan p = vis::plan_image(height, width); + int v[8]; + eight(p, v); + for (int x : v) + for (int b = 0; b < 4; ++b) h = (h ^ uint64_t((uint32_t(x) >> (8 * b)) & 0xff)) * 0x100000001b3ULL; + if (p.total_rows() > max_rows) max_rows = p.total_rows(); + ++pairs; + // what the tower needs of every piece: both patch extents multiples of 4, and at least one row + const auto ok = [](const vis::PiecePlan& q) { + return q.grid_h > 0 && q.grid_w > 0 && q.grid_h % 4 == 0 && q.grid_w % 4 == 0 && q.height == q.grid_h * 14 && + q.width == q.grid_w * 14 && q.rows == q.grid_h * q.grid_w / 16; + }; + shapes_ok = shapes_ok && ok(p.source) && (p.slice_rows == 0 || ok(p.slice)) && p.slice_rows * p.slice_cols <= 9 && + p.pieces() == 1 + p.slice_rows * p.slice_cols; + } + std::printf(" %d image sizes: hash 0x%016llx (reference 0x%016llx), most text rows %d\n", pairs, (unsigned long long)h, + (unsigned long long)kReferenceHash, max_rows); + check(h == kReferenceHash, "all 3,481 plans are the reference's (one hash over every number)"); + check(max_rows == kReferenceMaxRows, "the largest plan gives the reference's row count"); + check(shapes_ok, "every piece of every plan is a multiple of 4 patches in both extents, and at most 9 slices"); + + { + const vis::ImagePlan p = vis::plan_image(1080, 1920); + check(p.pieces() == 9 && p.source.rows == 6 * 11 && p.slice.rows == 64 && p.total_rows() == 66 + 8 * 64, + "1080 x 1920: one 24 x 44 source (66 rows) and eight 32 x 32 slices (64 rows each), 578 rows"); + } + for (const auto& [hh, ww] : std::vector>{{0, 100}, {100, 0}, {-1, 5}, {1 << 21, 10}}) { + std::string why; + try { vis::plan_image(hh, ww); } catch (const std::exception& e) { why = e.what(); } + check(why.find("cannot be planned") != std::string::npos, std::to_string(hh) + " x " + std::to_string(ww) + " is refused"); + } + std::printf("vision plan: %d checks, %d failed\n", g_arms, g_fail); + return g_fail ? 1 : 0; +} From b9ffa6a6730d269db60161e3a1996189c2743310 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 19:43:02 -0400 Subject: [PATCH 2/3] npu: MiniCPM-V 4.7 image bytes to the tower's input: torch's uint8 bicubic bit for bit, slices, normalise (engine#1788) vision::resize_bicubic_u8 reproduces torch.nn.functional.interpolate(uint8, bicubic, antialias) exactly (11 cases held as hashes of the reference's bytes); vision::preprocess returns the plan's pieces as normalised planes, whole view first. End to end on the 1B pack (460x470 synthetic image, 3 pieces, 196 rows) against the reference plan + torch resize + transformers tower: max |d| 7.5e-6, corr 1.000000000. 110 s on one thread: a reference, not a serving path. Test: npu_vision_image (42 checks; 6 mutants caught; +3 end to end with ONEBIT_MINICPM_PACK / ONEBIT_MINICPM_E2E). Co-Authored-By: Claude Fable 5.2 --- CMakeLists.txt | 7 +- docs/npu.md | 30 ++++- npu/vision_image.cpp | 154 +++++++++++++++++++++++++ npu/vision_image.h | 50 ++++++++ tests/npu_vision_image_test.cpp | 195 ++++++++++++++++++++++++++++++++ 5 files changed, 433 insertions(+), 3 deletions(-) create mode 100644 npu/vision_image.cpp create mode 100644 npu/vision_image.h create mode 100644 tests/npu_vision_image_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index d0b398b4a..b16cabee7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -693,7 +693,7 @@ option(ONEBIT_NPU_HOST "Build the NPU host library and its tests" ${_onebit_npu_ if(ONEBIT_NPU_HOST) find_package(PkgConfig REQUIRED) pkg_check_modules(PCRE2 REQUIRED IMPORTED_TARGET libpcre2-8) -add_library(onebit_npu STATIC npu/file_map.cpp npu/flm_head_import.cpp npu/full_elf.cpp npu/model.cpp npu/vision_pack.cpp npu/vision_tower.cpp +add_library(onebit_npu STATIC npu/file_map.cpp npu/flm_head_import.cpp npu/full_elf.cpp npu/model.cpp npu/vision_pack.cpp npu/vision_tower.cpp npu/vision_image.cpp npu/moe.cpp npu/pack.cpp npu/pack_import.cpp npu/private_route.cpp npu/q4nx.cpp npu/q4nx_device.cpp npu/tokenizer.cpp) target_include_directories(onebit_npu PUBLIC npu) @@ -1224,6 +1224,11 @@ add_executable(npu_vision_plan_test tests/npu_vision_plan_test.cpp) target_include_directories(npu_vision_plan_test PRIVATE npu) add_test(NAME npu_vision_plan COMMAND npu_vision_plan_test) add_dependencies(onebit npu_vision_plan_test) +# The resize (bit for bit against torch's uint8 antialiased bicubic) and the pieces handed to the tower. +add_executable(npu_vision_image_test tests/npu_vision_image_test.cpp) +target_link_libraries(npu_vision_image_test PRIVATE onebit_npu) +add_test(NAME npu_vision_image COMMAND npu_vision_image_test) +add_dependencies(onebit npu_vision_image_test) add_executable(npu_q4nx_test tests/npu_q4nx_test.cpp) target_include_directories(npu_q4nx_test PRIVATE app) target_link_libraries(npu_q4nx_test PRIVATE onebit_npu) diff --git a/docs/npu.md b/docs/npu.md index fff878f09..0090813fe 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1559,8 +1559,34 @@ resize. Every piece is a multiple of 56 pixels (4 patches) in both extents, beca (heights first.) So one 448-pixel slice is 64 rows, not the 1,225 that 980 / 14 / 2 squared suggests: the position table is 70 × 70, but no piece the processor produces is that large unless the image is a very long strip. The plan is checked against the reference's own answers for 3,481 image sizes in -`tests/npu_vision_plan_test.cpp`. Resampling the pixels (the reference uses bicubic), rescaling and the -slice separators on the text side are not written. +`tests/npu_vision_plan_test.cpp`. + +**From bytes to the tower's input (`npu/vision_image.h`).** `vision::preprocess(rgb, height, width, plan)` +resizes an RGB byte image to the plan's whole view and, when it is sliced, to the refine size, cuts the +slices, and returns each piece as `(3, height, width)` float32 with value `(byte − 127.5) / 127.5`: the +whole view first, then the slices row by row. The resize is bicubic with antialiasing on bytes and +reproduces `torch.nn.functional.interpolate(uint8, mode="bicubic", antialias=True)` (torch 2.13) bit +for bit: the a = −0.5 cubic stretched by the scale when shrinking, integer weights at the largest +precision that still fits int16 (PIL's fixed 22 bits differs by one in about 1 % of bytes), width +first and then height. `tests/npu_vision_image_test.cpp` holds the reference's bytes for eleven +resizes as hashes. + +End to end on the 1B pack, 2026-11-10: a 460 × 470 synthetic image through plan, resize, pieces and +`tower_forward()`, against the reference's plan, torch's resize and the transformers tower on the same +image: + +| piece | patches | rows | max \|d\| | corr | +|---|---|---|---|---| +| whole view 448 × 448 | 32 × 32 | 64 | 6.4e-6 | 1.000000000 | +| slice 0, 616 × 336 | 44 × 24 | 66 | 7.5e-6 | 1.000000000 | +| slice 1, 616 × 336 | 44 × 24 | 66 | 6.7e-6 | 1.000000000 | + +That run took 110 s on one thread, which is the host reference's cost and the reason it is not a +serving path. Two things are not established: the reference side called torch's `interpolate` +directly, on the understanding that this is what the processor's torchvision resize does for byte +images on a CPU build with AVX2 (torchvision is not on this box, so that link is read, not run); and +no real photograph or image decoder has been through it. The slice separators and placeholder rows on +the text side are not written. ## Serving diff --git a/npu/vision_image.cpp b/npu/vision_image.cpp new file mode 100644 index 000000000..bb6dbdce5 --- /dev/null +++ b/npu/vision_image.cpp @@ -0,0 +1,154 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "vision_image.h" + +#include +#include +#include + +namespace onebit::npu::vision { + +namespace { + +double cubic(double x) { + const double a = -0.5; + if (x < 0) x = -x; + if (x < 1.0) return ((a + 2.0) * x - (a + 3.0)) * x * x + 1.0; + if (x < 2.0) return (((x - 5.0) * x + 8.0) * x - 4.0) * a; + return 0.0; +} + +struct Taps { + std::vector first; // per output index: the first input index + std::vector count; // and how many taps + std::vector weight; // count taps each, `stride` apart + int stride = 0, precision = 0; +}; + +Taps taps_for(int in, int out) { + const double scale = double(in) / double(out); + const double fs = scale > 1.0 ? scale : 1.0, support = 2.0 * fs, inv = 1.0 / fs; + Taps t; + t.stride = int(std::ceil(support)) * 2 + 1; + t.first.resize(size_t(out)); + t.count.resize(size_t(out)); + std::vector w(size_t(out) * t.stride, 0.0); + double wmax = 0.0; + for (int i = 0; i < out; ++i) { + const double center = scale * (i + 0.5); + int lo = int(center - support + 0.5); + if (lo < 0) lo = 0; + int hi = int(center + support + 0.5); + if (hi > in) hi = in; + const int n = hi - lo; + double total = 0.0; + for (int j = 0; j < n; ++j) { + w[size_t(i) * t.stride + j] = cubic((j + lo - center + 0.5) * inv); + total += w[size_t(i) * t.stride + j]; + } + for (int j = 0; j < n; ++j) { + if (total != 0.0) w[size_t(i) * t.stride + j] /= total; + if (w[size_t(i) * t.stride + j] > wmax) wmax = w[size_t(i) * t.stride + j]; + } + t.first[size_t(i)] = lo; + t.count[size_t(i)] = n; + } + // the largest precision at which the biggest weight still fits a signed 16-bit integer + for (int p = 0; p < 22; ++p) { + if (int64_t(0.5 + wmax * double(int64_t(1) << (p + 1))) >= (1 << 15)) break; + t.precision = p + 1; + } + t.weight.resize(w.size()); + for (size_t k = 0; k < w.size(); ++k) { + const double v = w[k] * double(int64_t(1) << t.precision); + t.weight[k] = int32_t(v < 0 ? -0.5 + v : 0.5 + v); + } + return t; +} + +inline uint8_t clip8(int64_t v) { return uint8_t(v < 0 ? 0 : (v > 255 ? 255 : v)); } + +} // namespace + +std::vector resize_bicubic_u8(const uint8_t* rgb, int height, int width, int out_height, int out_width) { + if (!rgb || height <= 0 || width <= 0 || out_height <= 0 || out_width <= 0 || out_height > (1 << 16) || out_width > (1 << 16)) + throw std::runtime_error("cannot resize a " + std::to_string(height) + " x " + std::to_string(width) + " image to " + + std::to_string(out_height) + " x " + std::to_string(out_width)); + // width first + std::vector mid; + const uint8_t* src = rgb; + if (out_width != width) { + const Taps t = taps_for(width, out_width); + const int64_t half = t.precision > 0 ? int64_t(1) << (t.precision - 1) : 0; + mid.resize(size_t(height) * out_width * 3); + for (int y = 0; y < height; ++y) + for (int x = 0; x < out_width; ++x) { + const int32_t* w = t.weight.data() + size_t(x) * t.stride; + const uint8_t* p = rgb + (size_t(y) * width + size_t(t.first[size_t(x)])) * 3; + int64_t acc[3] = {half, half, half}; + for (int j = 0; j < t.count[size_t(x)]; ++j) + for (int c = 0; c < 3; ++c) acc[c] += int64_t(p[size_t(j) * 3 + c]) * w[j]; + for (int c = 0; c < 3; ++c) mid[(size_t(y) * out_width + x) * 3 + c] = clip8(acc[c] >> t.precision); + } + src = mid.data(); + } + if (out_height == height) return src == rgb ? std::vector(rgb, rgb + size_t(height) * width * 3) : mid; + // then height + const Taps t = taps_for(height, out_height); + const int64_t half = t.precision > 0 ? int64_t(1) << (t.precision - 1) : 0; + std::vector out(size_t(out_height) * out_width * 3); + const size_t row = size_t(out_width) * 3; + for (int y = 0; y < out_height; ++y) { + const int32_t* w = t.weight.data() + size_t(y) * t.stride; + for (size_t k = 0; k < row; ++k) { + int64_t acc = half; + for (int j = 0; j < t.count[size_t(y)]; ++j) acc += int64_t(src[(size_t(t.first[size_t(y)]) + j) * row + k]) * w[j]; + out[size_t(y) * row + k] = clip8(acc >> t.precision); + } + } + return out; +} + +std::vector preprocess(const uint8_t* rgb, int height, int width, const ImagePlan& plan) { + if (plan.source.height <= 0 || plan.source.width <= 0 || plan.source.grid_h <= 0 || plan.source.grid_w <= 0) + throw std::runtime_error("the image plan is empty"); + // one piece: the (h, w) window of an interleaved image at (y0, x0), as normalised planes + const auto planes = [](const std::vector& img, int img_w, int y0, int x0, const PiecePlan& q) { + Piece p; + p.grid_h = q.grid_h; + p.grid_w = q.grid_w; + p.pixels.resize(size_t(3) * q.height * q.width); + for (int c = 0; c < 3; ++c) + for (int y = 0; y < q.height; ++y) + for (int x = 0; x < q.width; ++x) + p.pixels[(size_t(c) * q.height + y) * q.width + x] = + (float(img[(size_t(y0 + y) * img_w + size_t(x0 + x)) * 3 + c]) - 127.5f) / 127.5f; + return p; + }; + std::vector out; + out.push_back(planes(resize_bicubic_u8(rgb, height, width, plan.source.height, plan.source.width), plan.source.width, 0, 0, + plan.source)); + if (plan.slice_rows > 0) { + if (plan.refine_height != plan.slice.height * plan.slice_rows || plan.refine_width != plan.slice.width * plan.slice_cols) + throw std::runtime_error("the image plan's slices do not tile its refine size"); + const std::vector refine = resize_bicubic_u8(rgb, height, width, plan.refine_height, plan.refine_width); + for (int r = 0; r < plan.slice_rows; ++r) + for (int c = 0; c < plan.slice_cols; ++c) + out.push_back(planes(refine, plan.refine_width, r * plan.slice.height, c * plan.slice.width, plan.slice)); + } + return out; +} + +} // namespace onebit::npu::vision diff --git a/npu/vision_image.h b/npu/vision_image.h new file mode 100644 index 000000000..e46b6a859 --- /dev/null +++ b/npu/vision_image.h @@ -0,0 +1,50 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// A MiniCPM-V 4.7 image from RGB bytes to what the vision tower takes (engine#1788): resize, slice, normalise. +// +// plan_image() (vision_plan.h) says which sizes; this does the pixels. The pieces come out in the order the reference +// processor feeds the tower: the whole view first, then the slices row by row. Each is (3, height, width) float32, +// value (byte - 127.5) / 127.5, i.e. rescale by 1/255 then mean 0.5 and std 0.5 (preprocessor_config.json). +// +// The resize is bicubic with antialiasing on BYTES, and it is torch's, bit for bit: torch.nn.functional.interpolate( +// uint8, mode="bicubic", antialias=True) at torch 2.13. That is integer arithmetic, so it can be exact, and three things +// about it are load bearing (tests/npu_vision_image_test.cpp has the reference's bytes): +// - the kernel is the a = -0.5 cubic, stretched by the scale when shrinking, normalised per output pixel; +// - weights become integers at the largest precision whose biggest weight still fits int16 (NOT PIL's fixed 22 bits: +// that differs from torch by one in about 1 % of bytes); +// - width first, then height, with the intermediate image rounded to bytes in between. A pass whose size does not +// change is skipped. +#pragma once + +#include "vision_plan.h" + +#include +#include + +namespace onebit::npu::vision { + +// `rgb` is (height, width, 3) interleaved bytes; the result is (out_height, out_width, 3). Throws on empty sizes. +std::vector resize_bicubic_u8(const uint8_t* rgb, int height, int width, int out_height, int out_width); + +struct Piece { + int grid_h = 0, grid_w = 0; // patches: what tower_forward() takes as gh, gw + std::vector pixels; // (3, grid_h * patch, grid_w * patch) +}; + +// Every piece of `plan` (which must be plan_image(height, width, ...) for this image), source first. +std::vector preprocess(const uint8_t* rgb, int height, int width, const ImagePlan& plan); + +} // namespace onebit::npu::vision diff --git a/tests/npu_vision_image_test.cpp b/tests/npu_vision_image_test.cpp new file mode 100644 index 000000000..f8638a03e --- /dev/null +++ b/tests/npu_vision_image_test.cpp @@ -0,0 +1,195 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// npu_vision_image_test +// +// The image path before the tower (npu/vision_image.h, engine#1788). +// +// resize_bicubic_u8 against torch 2.13's torch.nn.functional.interpolate(uint8, mode="bicubic", antialias=True): for +// each case the image below is resized by both, and the reference's bytes are held here as an FNV-1a hash of the whole +// output plus three sample bytes. The image is integer arithmetic, so it is the same picture in any language: +// byte(y, x, c) = (x*(7+c) + y*(13-c) + ((x*y) % 31)*5 + c*29 + ((x/9 + y/5) % 2)*90) & 255 +// preprocess(): the pieces, their order, and that each value is (byte - 127.5) / 127.5 of the resized image. +#include "../npu/vision_image.h" +#include "../npu/vision_tower.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +namespace vis = onebit::npu::vision; + +int g_fail = 0, g_arms = 0; +void check(bool ok, const std::string& what) { + ++g_arms; + if (!ok) { + ++g_fail; + std::printf("FAIL %s\n", what.c_str()); + } +} + +std::vector image(int h, int w) { + std::vector v(size_t(h) * w * 3); + for (int y = 0; y < h; ++y) + for (int x = 0; x < w; ++x) + for (int c = 0; c < 3; ++c) + v[(size_t(y) * w + x) * 3 + c] = + uint8_t((x * (7 + c) + y * (13 - c) + ((x * y) % 31) * 5 + c * 29 + ((x / 9 + y / 5) % 2) * 90) & 255); + return v; +} + +uint64_t fnv(const std::vector& b) { + uint64_t h = 0xcbf29ce484222325ULL; + for (uint8_t v : b) h = (h ^ v) * 0x100000001b3ULL; + return h; +} + +// {height, width, out height, out width, hash of the reference's output, {byte (0,0,0), byte (oh/2, ow/3, 1), byte (oh-1, ow-1, 2)}} +struct Case { + int h, w, oh, ow; + uint64_t hash; + int sample[3]; +}; +const Case kCases[] = { + {37, 53, 56, 112, 0x183132e60e013115ULL, {0, 52, 214}}, // up in both + {480, 640, 392, 504, 0xa3482dec42670b72ULL, {3, 169, 175}}, // down in both: a 480 x 640 image's whole view + {97, 211, 448, 448, 0x71de32691a84dd1fULL, {0, 202, 255}}, // up in one, up more in the other + {300, 200, 168, 112, 0xd94cee2b35fef077ULL, {9, 201, 197}}, + {64, 64, 56, 56, 0x909ccbe6090e462dULL, {2, 161, 119}}, // a slight shrink + {1000, 30, 224, 56, 0x1a51666a837eb165ULL, {25, 137, 139}}, // down 4.5x in one, up in the other + {50, 50, 50, 112, 0x5bf628c4db3c6e45ULL, {0, 71, 79}}, // height unchanged: that pass is skipped + {56, 56, 56, 56, 0x97d7e9db2a2df6b3ULL, {0, 127, 58}}, // nothing to do: the image comes back + {3, 5, 56, 56, 0xfe99d14c4912f969ULL, {0, 56, 162}}, // fewer input pixels than taps + {1, 1, 448, 448, 0x9053309dc853d325ULL, {0, 29, 58}}, + {480, 640, 560, 784, 0x4c864ee63453d9d4ULL, {0, 91, 158}}, // the same image's refine size (1 x 2 slices) +}; + +} // namespace + +int main() { + for (const Case& c : kCases) { + const std::vector in = image(c.h, c.w); + const std::vector out = vis::resize_bicubic_u8(in.data(), c.h, c.w, c.oh, c.ow); + const std::string name = std::to_string(c.h) + " x " + std::to_string(c.w) + " -> " + std::to_string(c.oh) + " x " + std::to_string(c.ow); + const bool sized = out.size() == size_t(c.oh) * c.ow * 3; + check(sized, name + ": the output has the asked size"); + if (!sized) continue; + const int s0 = out[0], s1 = out[(size_t(c.oh / 2) * c.ow + size_t(c.ow / 3)) * 3 + 1], + s2 = out[(size_t(c.oh - 1) * c.ow + size_t(c.ow - 1)) * 3 + 2]; + check(s0 == c.sample[0] && s1 == c.sample[1] && s2 == c.sample[2], + name + ": three sample bytes are the reference's (" + std::to_string(s0) + ", " + std::to_string(s1) + ", " + std::to_string(s2) + ")"); + check(fnv(out) == c.hash, name + ": every byte is the reference's"); + } + { + std::string why; + const std::vector in = image(4, 4); + try { vis::resize_bicubic_u8(in.data(), 4, 4, 0, 8); } catch (const std::exception& e) { why = e.what(); } + check(why.find("cannot resize") != std::string::npos, "an empty output size is refused"); + } + + // preprocess: a 480 x 640 image is one whole view (392 x 504) and 1 x 2 slices of 560 x 392 cut from 560 x 784 + { + const int h = 480, w = 640; + const std::vector in = image(h, w); + const vis::ImagePlan plan = vis::plan_image(h, w); + const std::vector pieces = vis::preprocess(in.data(), h, w, plan); + check(pieces.size() == 3 && plan.slice_rows == 1 && plan.slice_cols == 2, "480 x 640: the whole view and two slices"); + if (pieces.size() == 3) { + check(pieces[0].grid_h == 28 && pieces[0].grid_w == 36 && pieces[0].pixels.size() == size_t(3) * 392 * 504, + "the whole view comes first, 28 x 36 patches"); + check(pieces[1].grid_h == 40 && pieces[1].grid_w == 28 && pieces[2].pixels.size() == size_t(3) * 560 * 392, + "then the slices, 40 x 28 patches each"); + const std::vector whole = vis::resize_bicubic_u8(in.data(), h, w, 392, 504); + const std::vector refine = vis::resize_bicubic_u8(in.data(), h, w, 560, 784); + bool ok0 = true, ok1 = true, ok2 = true; + for (int c = 0; c < 3; ++c) + for (int y = 0; y < 392; y += 7) + for (int x = 0; x < 504; x += 5) + ok0 = ok0 && pieces[0].pixels[(size_t(c) * 392 + y) * 504 + x] == + (float(whole[(size_t(y) * 504 + x) * 3 + c]) - 127.5f) / 127.5f; + for (int c = 0; c < 3; ++c) + for (int y = 0; y < 560; y += 7) + for (int x = 0; x < 392; x += 5) { + ok1 = ok1 && pieces[1].pixels[(size_t(c) * 560 + y) * 392 + x] == + (float(refine[(size_t(y) * 784 + x) * 3 + c]) - 127.5f) / 127.5f; + ok2 = ok2 && pieces[2].pixels[(size_t(c) * 560 + y) * 392 + x] == + (float(refine[(size_t(y) * 784 + 392 + x) * 3 + c]) - 127.5f) / 127.5f; + } + check(ok0, "the whole view is the resized image as planes, (byte - 127.5) / 127.5"); + check(ok1, "slice 0 is the left half of the refine image"); + check(ok2, "slice 1 is the right half of the refine image"); + float lo = 2, hi = -2; + for (float v : pieces[0].pixels) { lo = v < lo ? v : lo; hi = v > hi ? v : hi; } + check(lo >= -1.0f && hi <= 1.0f && lo < -0.9f && hi > 0.9f, "values span [-1, 1]"); + } + } + // a small image: scaled up to one 448 x 448 view, no slices + { + const std::vector in = image(100, 100); + const std::vector pieces = vis::preprocess(in.data(), 100, 100, vis::plan_image(100, 100)); + check(pieces.size() == 1 && pieces[0].grid_h == 32 && pieces[0].grid_w == 32, "100 x 100: one 32 x 32-patch view"); + } + // End to end on a published pack: bytes -> plan -> resize -> pieces -> tower, against a reference run of the same + // image (the reference's plan, torch's resize, the transformers tower). ONEBIT_MINICPM_E2E holds image.txt + // ("height width") and out_.f32 for each piece. + const char* pack = std::getenv("ONEBIT_MINICPM_PACK"); + const char* e2e = std::getenv("ONEBIT_MINICPM_E2E"); + if (pack && *pack && e2e && *e2e) { + try { + int h = 0, w = 0; + { std::ifstream f(std::string(e2e) + "/image.txt"); f >> h >> w; } + const std::vector in = image(h, w); + const vis::ImagePlan plan = vis::plan_image(h, w); + const std::vector pieces = vis::preprocess(in.data(), h, w, plan); + const vis::Pack p(pack); + std::printf(" %d x %d image: %zu piece(s), %d text rows\n", h, w, pieces.size(), plan.total_rows()); + for (size_t k = 0; k < pieces.size(); ++k) { + const std::vector out = vis::tower_forward(p, pieces[k].pixels.data(), pieces[k].grid_h, pieces[k].grid_w); + std::vector want; + { + std::ifstream f(std::string(e2e) + "/out_" + std::to_string(k) + ".f32", std::ios::binary | std::ios::ate); + if (f) { + want.resize(size_t(f.tellg()) / 4); + f.seekg(0); + f.read(reinterpret_cast(want.data()), std::streamsize(want.size() * 4)); + } + } + double mx = 0, sa = 0, sb = 0, saa = 0, sbb = 0, sab = 0; + const size_t n = out.size() < want.size() ? out.size() : want.size(); + for (size_t i = 0; i < n; ++i) { + mx = std::fmax(mx, std::fabs(double(out[i]) - want[i])); + sa += out[i]; sb += want[i]; saa += double(out[i]) * out[i]; sbb += double(want[i]) * want[i]; sab += double(out[i]) * want[i]; + } + const double corr = n ? (sab - sa * sb / double(n)) / std::sqrt((saa - sa * sa / double(n)) * (sbb - sb * sb / double(n))) : 0; + std::printf(" piece %zu (%d x %d patches): %zu values, max |d| %.3g, corr %.9f\n", k, pieces[k].grid_h, pieces[k].grid_w, + out.size(), mx, corr); + check(out.size() == want.size() && !out.empty() && corr > 0.99999 && mx < 5e-3, + "published pack, piece " + std::to_string(k) + ": the rows are the reference's"); + } + } catch (const std::exception& e) { + check(false, std::string("the published pack runs end to end (threw: ") + e.what() + ")"); + } + } else { + std::printf(" (ONEBIT_MINICPM_PACK / ONEBIT_MINICPM_E2E not set: the end-to-end arm did not run)\n"); + } + std::printf("vision image: %d checks, %d failed\n", g_arms, g_fail); + return g_fail ? 1 : 0; +} From 824db208733cd35e2fe7c16736915acc6ef06e02 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 20:52:28 -0400 Subject: [PATCH 3/3] npu: the image plan says it never crops, the constants are pinned on known bytes, adversarial sizes (review of #2066) Review asks at 44b5438: (1) say and test what happens to sizes that are not multiples of anything: resized whole, never refused, never cropped; the slices tile the refine size exactly. (2) state the constants and pin one on known bytes: a one-colour (0, 255, 128) image comes out as planes of exactly -1, +1 and 0.5/127.5, in R, G, B order. (3) adversarial sizes: odd and off-by-one sizes, the largest accepted sizes (the reference's own answers for 2^20 strips), and one past the limit on each axis refused. Docs say the plan touches no file and no pixel, and does not bound its result. Co-Authored-By: Claude Fable 5.2 --- docs/npu.md | 12 +++++++++--- npu/vision_image.h | 3 ++- npu/vision_plan.h | 9 +++++++++ tests/npu_vision_image_test.cpp | 25 +++++++++++++++++++++++++ tests/npu_vision_plan_test.cpp | 19 +++++++++++++++++-- 5 files changed, 62 insertions(+), 6 deletions(-) diff --git a/docs/npu.md b/docs/npu.md index 0090813fe..0bae8a3c6 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1559,12 +1559,18 @@ resize. Every piece is a multiple of 56 pixels (4 patches) in both extents, beca (heights first.) So one 448-pixel slice is 64 rows, not the 1,225 that 980 / 14 / 2 squared suggests: the position table is 70 × 70, but no piece the processor produces is that large unless the image is a very long strip. The plan is checked against the reference's own answers for 3,481 image sizes in -`tests/npu_vision_plan_test.cpp`. +`tests/npu_vision_plan_test.cpp`. Nothing is cropped or padded and no size is refused for not being a +multiple of anything: every image is resized whole (385 × 384 becomes 448 × 448, so the aspect ratio moves +a little, as it does in the model's own processor), and the slices tile the refine size exactly. The plan +does not bound its result: a 1,048,576 × 1 strip gives the reference's answer, a piece 458,752 pixels +tall, and it is the caller that decides how many rows to accept. `plan_image()` reads no file and touches +no pixel, which is what lets it be checked anywhere; keep it that way. **From bytes to the tower's input (`npu/vision_image.h`).** `vision::preprocess(rgb, height, width, plan)` resizes an RGB byte image to the plan's whole view and, when it is sliced, to the refine size, cuts the -slices, and returns each piece as `(3, height, width)` float32 with value `(byte − 127.5) / 127.5`: the -whole view first, then the slices row by row. The resize is bicubic with antialiasing on bytes and +slices, and returns each piece as `(3, height, width)` float32 with value `(byte − 127.5) / 127.5` +(rescale by 1/255, then mean 0.5 and std 0.5, the same for each channel; R, G, B order in and out; byte +0 is exactly −1 and byte 255 exactly +1): the whole view first, then the slices row by row. The resize is bicubic with antialiasing on bytes and reproduces `torch.nn.functional.interpolate(uint8, mode="bicubic", antialias=True)` (torch 2.13) bit for bit: the a = −0.5 cubic stretched by the scale when shrinking, integer weights at the largest precision that still fits int16 (PIL's fixed 22 bits differs by one in about 1 % of bytes), width diff --git a/npu/vision_image.h b/npu/vision_image.h index e46b6a859..02a811ded 100644 --- a/npu/vision_image.h +++ b/npu/vision_image.h @@ -17,7 +17,8 @@ // // plan_image() (vision_plan.h) says which sizes; this does the pixels. The pieces come out in the order the reference // processor feeds the tower: the whole view first, then the slices row by row. Each is (3, height, width) float32, -// value (byte - 127.5) / 127.5, i.e. rescale by 1/255 then mean 0.5 and std 0.5 (preprocessor_config.json). +// value (byte - 127.5) / 127.5, i.e. rescale by 1/255 then mean 0.5 and std 0.5 (preprocessor_config.json), the same +// for all three channels. Input and planes are in R, G, B order: byte 0 is exactly -1, byte 255 exactly +1. // // The resize is bicubic with antialiasing on BYTES, and it is torch's, bit for bit: torch.nn.functional.interpolate( // uint8, mode="bicubic", antialias=True) at torch 2.13. That is integer arithmetic, so it can be exact, and three things diff --git a/npu/vision_plan.h b/npu/vision_plan.h index 9842b3838..873d2542c 100644 --- a/npu/vision_plan.h +++ b/npu/vision_plan.h @@ -29,6 +29,15 @@ // - an image that is not sliced is scaled UP to the resolution; one that is sliced has its whole view scaled down only. // (The per-slice size, refine / grid, is an exact division: the refine size was just rounded to a multiple of the grid.) // tests/npu_vision_plan_test.cpp holds that implementation's answers for 3,481 image sizes. +// +// Two properties a caller may rely on, both tested: +// - NOTHING IS CROPPED OR PADDED. No image size is refused for not being a multiple of anything: every size is +// resized, whole, to sizes that are multiples of 4 patches, and the slices tile the refine size with no remainder. +// The aspect ratio therefore moves a little (385 x 384 becomes 448 x 448); that is the model's own behaviour. +// - this header touches no file, no pixel and no platform: sizes in, sizes out. Keep it that way; it is what lets the +// plan be checked anywhere. +// It does not bound the result. A very long strip gives the reference's answer, which can be a piece hundreds of +// thousands of pixels long: the caller decides how many rows (total_rows()) it will accept. #pragma once #include diff --git a/tests/npu_vision_image_test.cpp b/tests/npu_vision_image_test.cpp index f8638a03e..6fc28fad9 100644 --- a/tests/npu_vision_image_test.cpp +++ b/tests/npu_vision_image_test.cpp @@ -141,6 +141,31 @@ int main() { check(lo >= -1.0f && hi <= 1.0f && lo < -0.9f && hi > 0.9f, "values span [-1, 1]"); } } + // The constants, on bytes whose answer is known without a reference: an image that is one colour, (0, 255, 128) in + // R, G, B order. Channel order, the 0..255 -> [-1, 1] range and the mean/std all show here: a swapped channel + // moves the planes, a 0..1 range leaves every value near -1, and a 1/128 divisor misses +1. + { + std::vector in(size_t(60) * 90 * 3); + for (size_t i = 0; i < in.size(); i += 3) { in[i] = 0; in[i + 1] = 255; in[i + 2] = 128; } + const std::vector big = vis::resize_bicubic_u8(in.data(), 60, 90, 448, 672); + bool flat = true; + for (size_t i = 0; i < big.size(); i += 3) flat = flat && big[i] == 0 && big[i + 1] == 255 && big[i + 2] == 128; + check(flat, "a one-colour image is the same colour after the resize, every byte"); + const vis::ImagePlan plan = vis::plan_image(60, 90); + const std::vector pieces = vis::preprocess(in.data(), 60, 90, plan); + bool r = !pieces.empty(), g = r, b = r; + if (!pieces.empty()) { + const size_t plane = pieces[0].pixels.size() / 3; + for (size_t i = 0; i < plane; ++i) { + r = r && pieces[0].pixels[i] == -1.0f; + g = g && pieces[0].pixels[plane + i] == 1.0f; + b = b && pieces[0].pixels[2 * plane + i] == (128.0f - 127.5f) / 127.5f; + } + } + check(r, "plane 0 is red: byte 0 is exactly -1"); + check(g, "plane 1 is green: byte 255 is exactly +1"); + check(b, "plane 2 is blue: byte 128 is 0.5 / 127.5"); + } // a small image: scaled up to one 448 x 448 view, no slices { const std::vector in = image(100, 100); diff --git a/tests/npu_vision_plan_test.cpp b/tests/npu_vision_plan_test.cpp index bce00c382..ca34e38af 100644 --- a/tests/npu_vision_plan_test.cpp +++ b/tests/npu_vision_plan_test.cpp @@ -65,6 +65,17 @@ const Row kRows[] = { {600, 800, {392, 504, 2, 2, 784, 1008, 392, 504}}, {8000, 8000, {448, 448, 3, 3, 1344, 1344, 448, 448}}, {14, 8000, {56, 10304, 0, 0, 0, 0, 0, 0}}, + // sizes that are not multiples of anything: resized, never refused, never cropped + {385, 384, {448, 448, 0, 0, 0, 0, 0, 0}}, + {447, 449, {448, 448, 0, 0, 0, 0, 0, 0}}, + {449, 449, {448, 448, 2, 1, 672, 616, 336, 616}}, + {1, 2, {336, 616, 0, 0, 0, 0, 0, 0}}, + // the largest sizes the plan accepts. The strips are the reference's answers too, absurd as they are (a 56-pixel + // wide piece 458,752 pixels tall): the plan is arithmetic, and it is the caller that bounds total_rows(). + {1048576, 1048576, {448, 448, 3, 3, 1344, 1344, 448, 448}}, + {1, 1048576, {56, 56, 1, 7, 56, 1048600, 56, 149800}}, + {1048576, 1, {458752, 56, 7, 1, 1213632, 56, 173376, 56}}, + {1048576, 3, {264880, 56, 9, 1, 794808, 56, 88312, 56}}, }; const uint64_t kReferenceHash = 0x8480087394bf204fULL; // the walk below, by the reference const int kReferenceMaxRows = 716; // the most text rows any of the 3,481 plans gives @@ -96,7 +107,7 @@ int main() { uint64_t h = 0xcbf29ce484222325ULL; int max_rows = 0, pairs = 0; - bool shapes_ok = true; + bool shapes_ok = true, tiles_ok = true; for (int height : kSizes) for (int width : kSizes) { const vis::ImagePlan p = vis::plan_image(height, width); @@ -113,19 +124,23 @@ int main() { }; shapes_ok = shapes_ok && ok(p.source) && (p.slice_rows == 0 || ok(p.slice)) && p.slice_rows * p.slice_cols <= 9 && p.pieces() == 1 + p.slice_rows * p.slice_cols; + // nothing is cropped or padded: the slices tile the refine size exactly, with no remainder on either axis + tiles_ok = tiles_ok && p.refine_height == p.slice.height * p.slice_rows && p.refine_width == p.slice.width * p.slice_cols; } std::printf(" %d image sizes: hash 0x%016llx (reference 0x%016llx), most text rows %d\n", pairs, (unsigned long long)h, (unsigned long long)kReferenceHash, max_rows); check(h == kReferenceHash, "all 3,481 plans are the reference's (one hash over every number)"); check(max_rows == kReferenceMaxRows, "the largest plan gives the reference's row count"); check(shapes_ok, "every piece of every plan is a multiple of 4 patches in both extents, and at most 9 slices"); + check(tiles_ok, "the slices of every plan tile its refine size exactly: no remainder to crop or pad"); { const vis::ImagePlan p = vis::plan_image(1080, 1920); check(p.pieces() == 9 && p.source.rows == 6 * 11 && p.slice.rows == 64 && p.total_rows() == 66 + 8 * 64, "1080 x 1920: one 24 x 44 source (66 rows) and eight 32 x 32 slices (64 rows each), 578 rows"); } - for (const auto& [hh, ww] : std::vector>{{0, 100}, {100, 0}, {-1, 5}, {1 << 21, 10}}) { + // one past the largest size on each axis, and the non-sizes + for (const auto& [hh, ww] : std::vector>{{0, 100}, {100, 0}, {-1, 5}, {1 << 21, 10}, {(1 << 20) + 1, 448}, {448, (1 << 20) + 1}}) { std::string why; try { vis::plan_image(hh, ww); } catch (const std::exception& e) { why = e.what(); } check(why.find("cannot be planned") != std::string::npos, std::to_string(hh) + " x " + std::to_string(ww) + " is refused");