diff --git a/CMakeLists.txt b/CMakeLists.txt index 71260bad5..4e417c464 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1767,6 +1767,12 @@ if(ONEBIT_NPU) target_include_directories(hybrid_prompt_probe PRIVATE ${CMAKE_SOURCE_DIR}/npu ${CMAKE_SOURCE_DIR}/npu/forward/include ${CMAKE_SOURCE_DIR}/npu/forward/engine-src) target_link_libraries(hybrid_prompt_probe PRIVATE onebit_npu_forward onebit_npu_rt) + # engine#1788: one picture and one question through a MiniCPM-V 4.7 pack with every step the engine's own + # (plan, resize, host tower, prompt tokens, canvas positions, then step_batch_embeds on the text model). + add_executable(minicpm_image_probe npu/forward/minicpm_image_probe.cpp) + target_include_directories(minicpm_image_probe PRIVATE ${CMAKE_SOURCE_DIR}/npu + ${CMAKE_SOURCE_DIR}/npu/forward/include ${CMAKE_SOURCE_DIR}/npu/forward/engine-src) + target_link_libraries(minicpm_image_probe PRIVATE onebit_npu onebit_npu_forward onebit_npu_rt) add_executable(mrope_planes_probe npu/forward/mrope_planes_probe.cpp) target_include_directories(mrope_planes_probe PRIVATE ${CMAKE_SOURCE_DIR}/npu ${CMAKE_SOURCE_DIR}/npu/forward/include ${CMAKE_SOURCE_DIR}/npu/forward/engine-src) diff --git a/docs/npu.md b/docs/npu.md index 43673cf4a..a54b87124 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1708,6 +1708,27 @@ prefixes, and none) as hashes over every token id and position. Images only: vid handled. Nothing feeds these planes or the tower's rows into the text forward yet; that is the route's work. +**One picture, one question, end to end (`minicpm_image_probe`).** A probe under `ONEBIT_NPU` that +chains every step above with nothing from outside the engine: + +``` +minicpm_image_probe (--image f.ppm | --solid R,G,B [--size HxW] | --halves R,G,B:R,G,B | --none) + --prompt "question" [--decode N] [--expect-ids ids.i32] +``` + +RGB bytes → `plan_image` → `preprocess` → `tower_forward` on the CPU → `image_tokens` and +`prompt_positions` → the text model's `step_batch_embeds()` with the tower's rows where the +placeholders are, then a greedy decode. The image is a binary PPM or a generated colour, so no image +decoder is involved, and the prompt is the one the pack's chat template renders for a single user turn +with one image and thinking off, written out in the probe rather than rendered. The text model opens +the device unless `NPU_INFER_HOST_GEMM` is set. + +It is a probe, not a route: `forward-serve` does not accept an image, and the tower's half minute to a +minute per 448 × 448 view on one thread is the host reference's cost. The measured rows are in +engine#1788. On the published packs the text model only loads once the padded-head rule of +engine#2026 is in: their head is stored 2 and 16 rows past the vocabulary, and without that rule the +probe stops at `text model: lm_head dequant failed`. + ## Serving The 1bit NPU engine runs the model on the XDNA 2 NPU. With a release package that carries the NPU diff --git a/npu/forward/minicpm_image_probe.cpp b/npu/forward/minicpm_image_probe.cpp new file mode 100644 index 000000000..cc161bdcb --- /dev/null +++ b/npu/forward/minicpm_image_probe.cpp @@ -0,0 +1,272 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// One picture and one question through a MiniCPM-V 4.7 pack, every step the engine's own (engine#1788): +// +// RGB bytes -> plan_image -> preprocess (resize, slices, normalise) -> tower_forward (host float32) per piece +// -> the prompt's tokens (image_tokens) and its three position planes (prompt_positions) +// -> the text model: step_batch_embeds() with the tower's rows where the placeholders are, then greedy decode. +// +// minicpm_image_probe (--image file.ppm | --solid R,G,B [--size HxW] | --halves R,G,B:R,G,B | --none) +// --prompt "question" [--decode N] [--expect-ids ids.i32] +// +// A probe, not a route: the tower runs on the CPU (about half a minute for one 448 x 448 view on one thread), the image is +// a binary PPM or a generated colour so that no image decoder is involved, and the prompt is the one the pack's own chat +// template renders for a single user turn with one image and thinking off, written out here rather than rendered: +// <|im_start|>user\n0[image]\n{question}<|im_end|>\n<|im_start|>assistant\n\n\n\n\n +// --expect-ids compares the token ids built here with a file of int32 ids (the reference processor's), and says so. +// The text model opens the device unless NPU_INFER_HOST_GEMM is set, so on the device this runs inside npu-hold. +#include "q4nx_forward.h" + +#include "tokenizer.h" +#include "vision_image.h" +#include "vision_pack.h" +#include "vision_plan.h" +#include "vision_prompt.h" +#include "vision_tower.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace vis = onebit::npu::vision; + +static bool parse_rgb(const std::string& s, uint8_t out[3]) { + int r = -1, g = -1, b = -1; + if (std::sscanf(s.c_str(), "%d,%d,%d", &r, &g, &b) != 3) return false; + if (r < 0 || r > 255 || g < 0 || g > 255 || b < 0 || b > 255) return false; + out[0] = uint8_t(r); + out[1] = uint8_t(g); + out[2] = uint8_t(b); + return true; +} + +// A binary PPM (P6, maxval 255): the one image format that needs no decoder. +static bool read_ppm(const std::string& path, std::vector& rgb, int& h, int& w) { + std::ifstream f(path, std::ios::binary); + if (!f) return false; + std::string magic; + int maxval = 0; + const auto token = [&]() { + std::string t; + for (;;) { + f >> t; + if (!f || t.empty() || t[0] != '#') break; + std::string rest; + std::getline(f, rest); + } + return t; + }; + magic = token(); + if (magic != "P6") return false; + w = std::atoi(token().c_str()); + h = std::atoi(token().c_str()); + maxval = std::atoi(token().c_str()); + f.get(); // the single whitespace byte after maxval + if (w <= 0 || h <= 0 || w > 16384 || h > 16384 || maxval != 255) return false; + rgb.resize(size_t(h) * w * 3); + f.read(reinterpret_cast(rgb.data()), std::streamsize(rgb.size())); + return f.gcount() == std::streamsize(rgb.size()); +} + +int main(int argc, char** argv) { + if (argc < 3) { + std::fprintf(stderr, "usage: %s (--image f.ppm | --solid R,G,B [--size HxW] | --halves R,G,B:R,G,B | " + "--none) --prompt \"question\" [--decode N] [--expect-ids ids.i32]\n", argv[0]); + return 2; + } + const std::string pack_dir = argv[1], kernels = argv[2]; + std::string image_path, prompt, expect_ids; + bool none = false, have_solid = false, have_halves = false; + uint8_t solid[3] = {0, 0, 0}, left[3] = {0, 0, 0}, right[3] = {0, 0, 0}; + int height = 448, width = 448, decode = 8; + for (int i = 3; i < argc; ++i) { + const std::string a = argv[i]; + const auto value = [&]() -> std::string { + if (i + 1 >= argc) { std::fprintf(stderr, "%s needs a value\n", a.c_str()); std::exit(2); } + return argv[++i]; + }; + if (a == "--image") image_path = value(); + else if (a == "--solid") { have_solid = parse_rgb(value(), solid); if (!have_solid) { std::fprintf(stderr, "--solid wants R,G,B\n"); return 2; } } + else if (a == "--halves") { + const std::string v = value(); + const size_t c = v.find(':'); + have_halves = c != std::string::npos && parse_rgb(v.substr(0, c), left) && parse_rgb(v.substr(c + 1), right); + if (!have_halves) { std::fprintf(stderr, "--halves wants R,G,B:R,G,B\n"); return 2; } + } else if (a == "--size") { if (std::sscanf(value().c_str(), "%dx%d", &height, &width) != 2 || height <= 0 || width <= 0 || height > 16384 || width > 16384) { std::fprintf(stderr, "--size wants HxW\n"); return 2; } } + else if (a == "--none") none = true; + else if (a == "--prompt") prompt = value(); + else if (a == "--decode") decode = std::atoi(value().c_str()); + else if (a == "--expect-ids") expect_ids = value(); + else { std::fprintf(stderr, "unknown option %s\n", a.c_str()); return 2; } + } + if (prompt.empty() || decode < 1 || decode > 512 || int(none) + int(have_solid) + int(have_halves) + int(!image_path.empty()) != 1) { + std::fprintf(stderr, "give --prompt and exactly one of --image, --solid, --halves, --none\n"); + return 2; + } + + try { + const auto now = [] { return std::chrono::steady_clock::now(); }; + const auto ms = [](std::chrono::steady_clock::time_point a, std::chrono::steady_clock::time_point b) { + return std::chrono::duration(b - a).count(); + }; + const onebit::npu::Tokenizer tok(pack_dir + "/tokenizer.json"); + vis::ImageMarkers m; + m.image_start = tok.token_id(""); + m.image_end = tok.token_id(""); + m.slice_start = tok.token_id(""); + m.slice_end = tok.token_id(""); + m.image_pad = tok.token_id("<|image_pad|>"); + const std::vector nl = tok.encode("\n"); + const int id_open = tok.token_id(""), id_close = tok.token_id(""); + if (m.image_start < 0 || m.image_end < 0 || m.slice_start < 0 || m.slice_end < 0 || m.image_pad < 0 || id_open < 0 || id_close < 0 || + nl.size() != 1) + throw std::runtime_error("this pack's tokenizer lacks one of , , , , <|image_pad|>, " + ", , or does not encode a newline as one token"); + m.newline = nl[0]; + + // ── the picture: bytes -> pieces -> the tower's rows ── + std::vector rows; // every piece's rows, in order, out_hidden wide + std::vector> grids; + std::vector image_ids; + int out_hidden = 0; + if (!none) { + std::vector rgb; + if (!image_path.empty()) { + if (!read_ppm(image_path, rgb, height, width)) throw std::runtime_error(image_path + " is not a binary PPM (P6, maxval 255)"); + } else { + rgb.resize(size_t(height) * width * 3); + for (int y = 0; y < height; ++y) + for (int x = 0; x < width; ++x) { + const uint8_t* c = have_solid ? solid : (x < width / 2 ? left : right); + std::memcpy(&rgb[(size_t(y) * width + x) * 3], c, 3); + } + } + const vis::Pack vp(pack_dir); + out_hidden = vp.geometry().out_hidden; + const vis::ImagePlan plan = vis::plan_image(height, width); + if (plan.total_rows() > 4096) throw std::runtime_error("this image would be " + std::to_string(plan.total_rows()) + " rows; the probe stops at 4096"); + const auto t0 = now(); + const std::vector pieces = vis::preprocess(rgb.data(), height, width, plan); + const auto t1 = now(); + for (const vis::Piece& p : pieces) { + const std::vector r = vis::tower_forward(vp, p.pixels.data(), p.grid_h, p.grid_w); + rows.insert(rows.end(), r.begin(), r.end()); + grids.push_back({p.grid_h, p.grid_w}); + } + const auto t2 = now(); + image_ids = vis::image_tokens(plan, m); + std::printf("image %d x %d: %zu piece(s), %d rows; preprocess %.0f ms, tower (host) %.0f ms\n", height, width, pieces.size(), + plan.total_rows(), ms(t0, t1), ms(t1, t2)); + } + + // ── the prompt, as the pack's template renders one user turn (thinking off) ── + std::vector ids = tok.encode("<|im_start|>user\n"); + if (!none) { + ids.push_back(id_open); + const std::vector zero = tok.encode("0"); + ids.insert(ids.end(), zero.begin(), zero.end()); + ids.push_back(id_close); + ids.insert(ids.end(), image_ids.begin(), image_ids.end()); + ids.push_back(m.newline); + } + const std::vector tail = tok.encode(prompt + "<|im_end|>\n<|im_start|>assistant\n\n\n\n\n"); + ids.insert(ids.end(), tail.begin(), tail.end()); + const vis::PromptPositions pos = vis::prompt_positions(ids, m, grids); + const int n = int(ids.size()); + if (!expect_ids.empty()) { + std::ifstream f(expect_ids, std::ios::binary | std::ios::ate); + std::vector want(f ? size_t(f.tellg()) / 4 : 0); + if (f) { f.seekg(0); f.read(reinterpret_cast(want.data()), std::streamsize(want.size() * 4)); } + bool same = want.size() == ids.size(); + for (size_t i = 0; same && i < ids.size(); ++i) same = want[i] == ids[i]; + std::printf("prompt ids vs %s: %s (%zu here, %zu there)\n", expect_ids.c_str(), same ? "IDENTICAL" : "DIFFERENT", ids.size(), want.size()); + } + + // ── the text model ── + Q4nxNpuForward f; + std::unique_ptr dev; + const std::string model = pack_dir + "/model.q4nx"; + bool ok; + const auto t3 = now(); + if (std::getenv("NPU_INFER_HOST_GEMM")) { + ok = f.init_host(model.c_str(), kernels.c_str()); + } else { + dev = std::make_unique(0); + ok = f.init(*dev, model.c_str(), kernels.c_str()); + } + if (!ok) throw std::runtime_error(std::string("text model: ") + f.last_error()); + std::vector one; + f.embed_row(ids[0], one); + const size_t H = one.size(); + if (!none && size_t(out_hidden) != H) throw std::runtime_error("the tower projects to " + std::to_string(out_hidden) + " but the text model is " + std::to_string(H) + " wide"); + std::vector embeds; + embeds.reserve(size_t(n) * H); + size_t next_row = 0; + for (int i = 0; i < n; ++i) { + if (ids[size_t(i)] == m.image_pad) { + if ((next_row + 1) * H > rows.size()) throw std::runtime_error("the prompt holds more placeholders than the tower produced rows"); + embeds.insert(embeds.end(), rows.begin() + long(next_row * H), rows.begin() + long((next_row + 1) * H)); + ++next_row; + } else { + f.embed_row(ids[size_t(i)], one); + embeds.insert(embeds.end(), one.begin(), one.end()); + } + } + if (next_row * H != rows.size()) throw std::runtime_error("the tower produced more rows than the prompt has placeholders"); + std::vector planes; + planes.reserve(size_t(3) * n); + planes.insert(planes.end(), pos.t.begin(), pos.t.end()); + planes.insert(planes.end(), pos.h.begin(), pos.h.end()); + planes.insert(planes.end(), pos.w.begin(), pos.w.end()); + Q4nxNpuForward::HybridInputs in; + in.position_ids = &planes; + std::vector logits; + f.reset(); + const auto t4 = now(); + if (!f.step_batch_embeds(embeds, 0, logits, in)) throw std::runtime_error(std::string("prefill: ") + f.last_error()); + const auto t5 = now(); + const auto argmax = [](const std::vector& v) { + int b = 0; + for (size_t i = 1; i < v.size(); ++i) + if (v[i] > v[size_t(b)]) b = int(i); + return b; + }; + const int im_end = tok.token_id("<|im_end|>"); + std::vector out; + out.push_back(argmax(logits)); + // the rotary position resumes after the canvas (pos.next); the cache slot is the prompt's length + for (int i = 1; i < decode && out.back() != im_end; ++i) { + if (!f.step(out.back(), pos.next + i - 1, logits, n + i - 1)) throw std::runtime_error(std::string("step: ") + f.last_error()); + out.push_back(argmax(logits)); + } + const auto t6 = now(); + std::printf("prompt %d tokens (%zu image rows), last position %d; init %.0f ms, prefill %.0f ms, %zu token(s) in %.0f ms\n", n, next_row, + pos.next - 1, ms(t3, t4), ms(t4, t5), out.size(), ms(t5, t6)); + std::printf("ids:"); + for (int t : out) std::printf(" %d", t); + std::printf("\nanswer: %s\n", tok.decode(out).c_str()); + return 0; + } catch (const std::exception& e) { + std::fprintf(stderr, "minicpm_image_probe: %s\n", e.what()); + return 1; + } +}