Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1767,6 +1767,12 @@ if(ONEBIT_NPU)
target_include_directories(hybrid_prompt_probe PRIVATE ${CMAKE_SOURCE_DIR}/npu
${CMAKE_SOURCE_DIR}/npu/forward/include ${CMAKE_SOURCE_DIR}/npu/forward/engine-src)
target_link_libraries(hybrid_prompt_probe PRIVATE onebit_npu_forward onebit_npu_rt)
# engine#1788: one picture and one question through a MiniCPM-V 4.7 pack with every step the engine's own
# (plan, resize, host tower, prompt tokens, canvas positions, then step_batch_embeds on the text model).
add_executable(minicpm_image_probe npu/forward/minicpm_image_probe.cpp)
target_include_directories(minicpm_image_probe PRIVATE ${CMAKE_SOURCE_DIR}/npu
${CMAKE_SOURCE_DIR}/npu/forward/include ${CMAKE_SOURCE_DIR}/npu/forward/engine-src)
target_link_libraries(minicpm_image_probe PRIVATE onebit_npu onebit_npu_forward onebit_npu_rt)
add_executable(mrope_planes_probe npu/forward/mrope_planes_probe.cpp)
target_include_directories(mrope_planes_probe PRIVATE ${CMAKE_SOURCE_DIR}/npu
${CMAKE_SOURCE_DIR}/npu/forward/include ${CMAKE_SOURCE_DIR}/npu/forward/engine-src)
Expand Down
21 changes: 21 additions & 0 deletions docs/npu.md
Original file line number Diff line number Diff line change
Expand Up @@ -1708,6 +1708,27 @@ prefixes, and none) as hashes over every token id and position. Images only: vid
handled. Nothing feeds these planes or the tower's rows into the text forward yet; that is the route's
work.

**One picture, one question, end to end (`minicpm_image_probe`).** A probe under `ONEBIT_NPU` that
chains every step above with nothing from outside the engine:

```
minicpm_image_probe <pack dir> <kernels dir> (--image f.ppm | --solid R,G,B [--size HxW] | --halves R,G,B:R,G,B | --none)
--prompt "question" [--decode N] [--expect-ids ids.i32]
```

RGB bytes → `plan_image` → `preprocess` → `tower_forward` on the CPU → `image_tokens` and
`prompt_positions` → the text model's `step_batch_embeds()` with the tower's rows where the
placeholders are, then a greedy decode. The image is a binary PPM or a generated colour, so no image
decoder is involved, and the prompt is the one the pack's chat template renders for a single user turn
with one image and thinking off, written out in the probe rather than rendered. The text model opens
the device unless `NPU_INFER_HOST_GEMM` is set.

It is a probe, not a route: `forward-serve` does not accept an image, and the tower's half minute to a
minute per 448 × 448 view on one thread is the host reference's cost. The measured rows are in
engine#1788. On the published packs the text model only loads once the padded-head rule of
engine#2026 is in: their head is stored 2 and 16 rows past the vocabulary, and without that rule the
probe stops at `text model: lm_head dequant failed`.

## Serving

The 1bit NPU engine runs the model on the XDNA 2 NPU. With a release package that carries the NPU
Expand Down
272 changes: 272 additions & 0 deletions npu/forward/minicpm_image_probe.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,272 @@
// Copyright 2026 bong-water-water-bong
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// One picture and one question through a MiniCPM-V 4.7 pack, every step the engine's own (engine#1788):
//
// RGB bytes -> plan_image -> preprocess (resize, slices, normalise) -> tower_forward (host float32) per piece
// -> the prompt's tokens (image_tokens) and its three position planes (prompt_positions)
// -> the text model: step_batch_embeds() with the tower's rows where the placeholders are, then greedy decode.
//
// minicpm_image_probe <pack dir> <kernels dir> (--image file.ppm | --solid R,G,B [--size HxW] | --halves R,G,B:R,G,B | --none)
// --prompt "question" [--decode N] [--expect-ids ids.i32]
//
// A probe, not a route: the tower runs on the CPU (about half a minute for one 448 x 448 view on one thread), the image is
// a binary PPM or a generated colour so that no image decoder is involved, and the prompt is the one the pack's own chat
// template renders for a single user turn with one image and thinking off, written out here rather than rendered:
// <|im_start|>user\n<image_id>0</image_id>[image]\n{question}<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n
// --expect-ids compares the token ids built here with a file of int32 ids (the reference processor's), and says so.
// The text model opens the device unless NPU_INFER_HOST_GEMM is set, so on the device this runs inside npu-hold.
#include "q4nx_forward.h"

#include "tokenizer.h"
#include "vision_image.h"
#include "vision_pack.h"
#include "vision_plan.h"
#include "vision_prompt.h"
#include "vision_tower.h"

#include <chrono>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <memory>
#include <stdexcept>
#include <string>
#include <vector>

namespace vis = onebit::npu::vision;

static bool parse_rgb(const std::string& s, uint8_t out[3]) {
int r = -1, g = -1, b = -1;
if (std::sscanf(s.c_str(), "%d,%d,%d", &r, &g, &b) != 3) return false;
if (r < 0 || r > 255 || g < 0 || g > 255 || b < 0 || b > 255) return false;
out[0] = uint8_t(r);
out[1] = uint8_t(g);
out[2] = uint8_t(b);
return true;
}

// A binary PPM (P6, maxval 255): the one image format that needs no decoder.
static bool read_ppm(const std::string& path, std::vector<uint8_t>& rgb, int& h, int& w) {
std::ifstream f(path, std::ios::binary);
if (!f) return false;
std::string magic;
int maxval = 0;
const auto token = [&]() {
std::string t;
for (;;) {
f >> t;
if (!f || t.empty() || t[0] != '#') break;
std::string rest;
std::getline(f, rest);
}
return t;
};
magic = token();
if (magic != "P6") return false;
w = std::atoi(token().c_str());
h = std::atoi(token().c_str());
maxval = std::atoi(token().c_str());
f.get(); // the single whitespace byte after maxval
if (w <= 0 || h <= 0 || w > 16384 || h > 16384 || maxval != 255) return false;
rgb.resize(size_t(h) * w * 3);
f.read(reinterpret_cast<char*>(rgb.data()), std::streamsize(rgb.size()));
return f.gcount() == std::streamsize(rgb.size());
}

int main(int argc, char** argv) {
if (argc < 3) {
std::fprintf(stderr, "usage: %s <pack dir> <kernels dir> (--image f.ppm | --solid R,G,B [--size HxW] | --halves R,G,B:R,G,B | "
"--none) --prompt \"question\" [--decode N] [--expect-ids ids.i32]\n", argv[0]);
return 2;
}
const std::string pack_dir = argv[1], kernels = argv[2];
std::string image_path, prompt, expect_ids;
bool none = false, have_solid = false, have_halves = false;
uint8_t solid[3] = {0, 0, 0}, left[3] = {0, 0, 0}, right[3] = {0, 0, 0};
int height = 448, width = 448, decode = 8;
for (int i = 3; i < argc; ++i) {
const std::string a = argv[i];
const auto value = [&]() -> std::string {
if (i + 1 >= argc) { std::fprintf(stderr, "%s needs a value\n", a.c_str()); std::exit(2); }
return argv[++i];
};
if (a == "--image") image_path = value();
else if (a == "--solid") { have_solid = parse_rgb(value(), solid); if (!have_solid) { std::fprintf(stderr, "--solid wants R,G,B\n"); return 2; } }
else if (a == "--halves") {
const std::string v = value();
const size_t c = v.find(':');
have_halves = c != std::string::npos && parse_rgb(v.substr(0, c), left) && parse_rgb(v.substr(c + 1), right);
if (!have_halves) { std::fprintf(stderr, "--halves wants R,G,B:R,G,B\n"); return 2; }
} else if (a == "--size") { if (std::sscanf(value().c_str(), "%dx%d", &height, &width) != 2 || height <= 0 || width <= 0 || height > 16384 || width > 16384) { std::fprintf(stderr, "--size wants HxW\n"); return 2; } }
else if (a == "--none") none = true;
else if (a == "--prompt") prompt = value();
else if (a == "--decode") decode = std::atoi(value().c_str());
else if (a == "--expect-ids") expect_ids = value();
else { std::fprintf(stderr, "unknown option %s\n", a.c_str()); return 2; }
}
if (prompt.empty() || decode < 1 || decode > 512 || int(none) + int(have_solid) + int(have_halves) + int(!image_path.empty()) != 1) {
std::fprintf(stderr, "give --prompt and exactly one of --image, --solid, --halves, --none\n");
return 2;
}

try {
const auto now = [] { return std::chrono::steady_clock::now(); };
const auto ms = [](std::chrono::steady_clock::time_point a, std::chrono::steady_clock::time_point b) {
return std::chrono::duration<double, std::milli>(b - a).count();
};
const onebit::npu::Tokenizer tok(pack_dir + "/tokenizer.json");
vis::ImageMarkers m;
m.image_start = tok.token_id("<image>");
m.image_end = tok.token_id("</image>");
m.slice_start = tok.token_id("<slice>");
m.slice_end = tok.token_id("</slice>");
m.image_pad = tok.token_id("<|image_pad|>");
const std::vector<int> nl = tok.encode("\n");
const int id_open = tok.token_id("<image_id>"), id_close = tok.token_id("</image_id>");
if (m.image_start < 0 || m.image_end < 0 || m.slice_start < 0 || m.slice_end < 0 || m.image_pad < 0 || id_open < 0 || id_close < 0 ||
nl.size() != 1)
throw std::runtime_error("this pack's tokenizer lacks one of <image>, </image>, <slice>, </slice>, <|image_pad|>, "
"<image_id>, </image_id>, or does not encode a newline as one token");
m.newline = nl[0];

// ── the picture: bytes -> pieces -> the tower's rows ──
std::vector<float> rows; // every piece's rows, in order, out_hidden wide
std::vector<std::pair<int, int>> grids;
std::vector<int> image_ids;
int out_hidden = 0;
if (!none) {
std::vector<uint8_t> rgb;
if (!image_path.empty()) {
if (!read_ppm(image_path, rgb, height, width)) throw std::runtime_error(image_path + " is not a binary PPM (P6, maxval 255)");
} else {
rgb.resize(size_t(height) * width * 3);
for (int y = 0; y < height; ++y)
for (int x = 0; x < width; ++x) {
const uint8_t* c = have_solid ? solid : (x < width / 2 ? left : right);
std::memcpy(&rgb[(size_t(y) * width + x) * 3], c, 3);
}
}
const vis::Pack vp(pack_dir);
out_hidden = vp.geometry().out_hidden;
const vis::ImagePlan plan = vis::plan_image(height, width);
if (plan.total_rows() > 4096) throw std::runtime_error("this image would be " + std::to_string(plan.total_rows()) + " rows; the probe stops at 4096");
const auto t0 = now();
const std::vector<vis::Piece> pieces = vis::preprocess(rgb.data(), height, width, plan);
const auto t1 = now();
for (const vis::Piece& p : pieces) {
const std::vector<float> r = vis::tower_forward(vp, p.pixels.data(), p.grid_h, p.grid_w);
rows.insert(rows.end(), r.begin(), r.end());
grids.push_back({p.grid_h, p.grid_w});
}
const auto t2 = now();
image_ids = vis::image_tokens(plan, m);
std::printf("image %d x %d: %zu piece(s), %d rows; preprocess %.0f ms, tower (host) %.0f ms\n", height, width, pieces.size(),
plan.total_rows(), ms(t0, t1), ms(t1, t2));
}

// ── the prompt, as the pack's template renders one user turn (thinking off) ──
std::vector<int> ids = tok.encode("<|im_start|>user\n");
if (!none) {
ids.push_back(id_open);
const std::vector<int> zero = tok.encode("0");
ids.insert(ids.end(), zero.begin(), zero.end());
ids.push_back(id_close);
ids.insert(ids.end(), image_ids.begin(), image_ids.end());
ids.push_back(m.newline);
}
const std::vector<int> tail = tok.encode(prompt + "<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n");
ids.insert(ids.end(), tail.begin(), tail.end());
const vis::PromptPositions pos = vis::prompt_positions(ids, m, grids);
const int n = int(ids.size());
if (!expect_ids.empty()) {
std::ifstream f(expect_ids, std::ios::binary | std::ios::ate);
std::vector<int32_t> want(f ? size_t(f.tellg()) / 4 : 0);
if (f) { f.seekg(0); f.read(reinterpret_cast<char*>(want.data()), std::streamsize(want.size() * 4)); }
bool same = want.size() == ids.size();
for (size_t i = 0; same && i < ids.size(); ++i) same = want[i] == ids[i];
std::printf("prompt ids vs %s: %s (%zu here, %zu there)\n", expect_ids.c_str(), same ? "IDENTICAL" : "DIFFERENT", ids.size(), want.size());
}

// ── the text model ──
Q4nxNpuForward f;
std::unique_ptr<xrt::device> dev;
const std::string model = pack_dir + "/model.q4nx";
bool ok;
const auto t3 = now();
if (std::getenv("NPU_INFER_HOST_GEMM")) {
ok = f.init_host(model.c_str(), kernels.c_str());
} else {
dev = std::make_unique<xrt::device>(0);
ok = f.init(*dev, model.c_str(), kernels.c_str());
}
if (!ok) throw std::runtime_error(std::string("text model: ") + f.last_error());
std::vector<float> one;
f.embed_row(ids[0], one);
const size_t H = one.size();
if (!none && size_t(out_hidden) != H) throw std::runtime_error("the tower projects to " + std::to_string(out_hidden) + " but the text model is " + std::to_string(H) + " wide");
std::vector<float> embeds;
embeds.reserve(size_t(n) * H);
size_t next_row = 0;
for (int i = 0; i < n; ++i) {
if (ids[size_t(i)] == m.image_pad) {
if ((next_row + 1) * H > rows.size()) throw std::runtime_error("the prompt holds more placeholders than the tower produced rows");
embeds.insert(embeds.end(), rows.begin() + long(next_row * H), rows.begin() + long((next_row + 1) * H));
++next_row;
} else {
f.embed_row(ids[size_t(i)], one);
embeds.insert(embeds.end(), one.begin(), one.end());
}
}
if (next_row * H != rows.size()) throw std::runtime_error("the tower produced more rows than the prompt has placeholders");
std::vector<int> planes;
planes.reserve(size_t(3) * n);
planes.insert(planes.end(), pos.t.begin(), pos.t.end());
planes.insert(planes.end(), pos.h.begin(), pos.h.end());
planes.insert(planes.end(), pos.w.begin(), pos.w.end());
Q4nxNpuForward::HybridInputs in;
in.position_ids = &planes;
std::vector<float> logits;
f.reset();
const auto t4 = now();
if (!f.step_batch_embeds(embeds, 0, logits, in)) throw std::runtime_error(std::string("prefill: ") + f.last_error());
const auto t5 = now();
const auto argmax = [](const std::vector<float>& v) {
int b = 0;
for (size_t i = 1; i < v.size(); ++i)
if (v[i] > v[size_t(b)]) b = int(i);
return b;
};
const int im_end = tok.token_id("<|im_end|>");
std::vector<int> out;
out.push_back(argmax(logits));
// the rotary position resumes after the canvas (pos.next); the cache slot is the prompt's length
for (int i = 1; i < decode && out.back() != im_end; ++i) {
if (!f.step(out.back(), pos.next + i - 1, logits, n + i - 1)) throw std::runtime_error(std::string("step: ") + f.last_error());
out.push_back(argmax(logits));
}
const auto t6 = now();
std::printf("prompt %d tokens (%zu image rows), last position %d; init %.0f ms, prefill %.0f ms, %zu token(s) in %.0f ms\n", n, next_row,
pos.next - 1, ms(t3, t4), ms(t4, t5), out.size(), ms(t5, t6));
std::printf("ids:");
for (int t : out) std::printf(" %d", t);
std::printf("\nanswer: %s\n", tok.decode(out).c_str());
return 0;
} catch (const std::exception& e) {
std::fprintf(stderr, "minicpm_image_probe: %s\n", e.what());
return 1;
}
}
Loading