Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
166 changes: 166 additions & 0 deletions app/npu_image_part.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,166 @@
// Copyright 2026 bong-water-water-bong
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// npu_image_part.h -- turning an OpenAI `image_url` content part into the bytes the vision chain wants (engine#2324).
//
// The chain downstream of here is decoder-agnostic: npu/vision_image.h's preprocess() takes raw RGB plus its geometry, and
// npu/vision_tower.h's tower_forward() takes the same. What the serve route had no way to produce was those bytes, because an
// image part arrives as a data: URL and nothing in this tree decodes one.
//
// This decodes exactly one media type -- **image/x-portable-pixmap, P6, maxval 255** -- and refuses the rest *by name*, which is
// the same discipline as the route's refusal: a PNG handed to a route with no PNG decoder must not become "an image that produced
// nothing". P6 is the format npu/forward/minicpm_image_probe.cpp already reads, so a picture that works in the probe works here,
// and adding a real decoder later is a change to this one function.
//
// usage:
// DecodedImage img;
// if (!decode_image_data_url(url, img)) -> 400 with img.error (it names the media type)
// preprocess(img.rgb.data(), img.height, img.width, plan)

#pragma once

#include <cstdint>
#include <cstring>
#include <string>
#include <vector>

namespace onebit::npu {

struct DecodedImage {
std::vector<uint8_t> rgb; // height * width * 3, row-major
int height = 0, width = 0;
std::string error; // set when the decode failed; names the media type when that was the reason
};

// RFC 4648 base64. Not a general-purpose encoder: the decoder is incremental-free and stops at the first bad character, which
// is what a request body needs -- anything after an invalid byte is not a picture.
inline bool base64_decode(const std::string& in, std::vector<uint8_t>& out) {
static const int8_t kMap[256] = {
-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,
-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,62,-1,-1,-1,63, 52,53,54,55,56,57,58,59,60,61,-1,-1,-1,-1,-1,-1,
-1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14, 15,16,17,18,19,20,21,22,23,24,25,-1,-1,-1,-1,-1,
-1,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40, 41,42,43,44,45,46,47,48,49,50,51,-1,-1,-1,-1,-1,
-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,
-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,
-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,
-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,
};
out.clear();
uint32_t acc = 0;
int bits = 0;
for (unsigned char ch : in) {
if (ch == '=') break; // padding ends the stream
const int v = kMap[ch];
if (v < 0) return false;
acc = (acc << 6) | uint32_t(v);
bits += 6;
if (bits >= 8) {
bits -= 8;
out.push_back(uint8_t((acc >> bits) & 0xffu));
}
}
return true;
}

// `data:image/x-portable-pixmap;base64,<payload>`, or `data:;base64,...` (the media type may be omitted, as browsers allow).
inline bool decode_image_data_url(const std::string& url, DecodedImage& out) {
out = DecodedImage{};
if (url.rfind("data:", 0) != 0) {
out.error = "the image part is not a data: URL; this route does not fetch remote images (engine#2324)";
return false;
}
const size_t comma = url.find(',');
if (comma == std::string::npos) {
out.error = "the image part has no ',' after its data: header";
return false;
}
const std::string header = url.substr(5, comma - 5);
const bool b64 = header.find(";base64") != std::string::npos;
if (!b64) {
out.error = "the image part is a data: URL that is not base64-encoded; only ;base64 is accepted (engine#2324)";
return false;
}
std::string media = header.substr(0, header.find(';'));
if (media.empty()) media = "image/x-portable-pixmap"; // `data:;base64,` means "you decide"
if (media != "image/x-portable-pixmap" && media != "image/x-ppm" && media != "image/ppm") {
out.error = "no decoder for media type '" + media + "': this route decodes image/x-portable-pixmap (P6) only, and "
"refuses the rest by name rather than answering without the picture (engine#2324)";
return false;
}
std::vector<uint8_t> raw;
if (!base64_decode(url.substr(comma + 1), raw)) {
out.error = "the image part's base64 payload is not valid base64";
return false;
}
// P6: "P6" whitespace W H whitespace 255 single-whitespace then binary RGB. Comments ('#') may appear in the header.
size_t i = 0;
auto skip_ws_and_comments = [&]() {
while (i < raw.size()) {
if (raw[i] == '#') { while (i < raw.size() && raw[i] != '\n') ++i; }
else if (raw[i] == ' ' || raw[i] == '\t' || raw[i] == '\r' || raw[i] == '\n') ++i;
else break;
}
};
// Accumulated in int64_t and bounded BEFORE it can overflow an int: the input is a remote data: URL, so a header
// claiming a width of 10^14 is a hostile value, not a typo. `int` arithmetic here was undefined behaviour that happened
// to be caught afterwards by the `w <= 0` check -- a parser must not rely on the wrap landing somewhere detectable.
bool range_error = false;
auto read_uint = [&](int& v) {
skip_ws_and_comments();
if (i >= raw.size() || raw[i] < '0' || raw[i] > '9') return false;
int64_t acc = 0;
while (i < raw.size() && raw[i] >= '0' && raw[i] <= '9') {
acc = acc * 10 + (raw[i] - '0');
++i;
if (acc > 2147483647LL) { range_error = true; return false; }
}
v = int(acc);
return true;
};
if (raw.size() < 2 || raw[0] != 'P' || raw[1] != '6') {
out.error = "the decoded image is not a P6 PPM ('P6' magic missing)";
return false;
}
i = 2;
int h = 0, w = 0, maxval = 0;
if (!read_uint(w) || !read_uint(h) || !read_uint(maxval)) {
out.error = range_error ? "a PPM header field is out of range (a dimension is larger than 2^31-1)"
: "the PPM header is truncated (width, height and maxval are required)";
return false;
}
if (maxval != 255) {
out.error = "the PPM has maxval " + std::to_string(maxval) + "; only 255 (one byte per sample) is accepted";
return false;
}
if (w <= 0 || h <= 0) {
out.error = "the PPM is " + std::to_string(w) + "x" + std::to_string(h) + ", which is not an image";
return false;
}
++i; // exactly one whitespace byte after maxval, then binary samples
// "At least", not "exactly": a PPM stream may carry trailing data after the samples, and refusing it would reject files
// that are correct. What the buffer handed downstream is -- exactly w*h*3 bytes -- is what has to be exact, and that is
// the `assign` below rather than this test.
const size_t need = size_t(w) * size_t(h) * 3u;
if (raw.size() < i + need) {
out.error = "the PPM is short: " + std::to_string(raw.size() - i) + " of " + std::to_string(need) + " sample bytes";
return false;
}
out.rgb.assign(raw.begin() + long(i), raw.begin() + long(i + need));
out.height = h;
out.width = w;
return true;
}

} // namespace onebit::npu
21 changes: 21 additions & 0 deletions cmake/tests-image-part.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
# Copyright 2026 bong-water-water-bong
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# engine#2324: the one place that knows what a picture arrives as -- a data: URL -- turning it into the raw RGB plus geometry
# that npu/vision_image.h and npu/vision_tower.h already take. Header-only, no model, no device.
add_executable(npu_image_part_test tests/npu_image_part_test.cpp)
target_include_directories(npu_image_part_test PRIVATE app)
add_test(NAME npu_image_part COMMAND npu_image_part_test)
add_dependencies(onebit npu_image_part_test)
155 changes: 155 additions & 0 deletions tests/npu_image_part_test.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
// Copyright 2026 bong-water-water-bong
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// npu_image_part_test.cpp -- the data: URL decode in front of the vision chain (engine#2324).
//
// Downstream of this function the chain is decoder-agnostic (npu/vision_image.h takes raw RGB plus geometry), so this is the one
// place that knows what a picture arrives as. Both directions are pinned: a P6 PPM decodes to exactly its samples, and every way
// of not being one -- a PNG, a remote URL, a truncated header, a short payload, maxval 16, a 0x0 image -- is refused with a
// message, because "an image that produced nothing" is the failure this file exists to prevent.
#include <cstdio>
#include <string>
#include <vector>

#include "npu_image_part.h"

static int g_fail = 0, g_n = 0;
static void check(bool ok, const std::string& what) {
++g_n;
if (!ok) { std::printf("FAIL %s\n", what.c_str()); ++g_fail; }
}

static std::string b64(const std::string& raw) {
static const char* t = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
std::string out;
uint32_t acc = 0; int bits = 0;
for (unsigned char c : raw) {
acc = (acc << 8) | c; bits += 8;
while (bits >= 6) { bits -= 6; out += t[(acc >> bits) & 0x3f]; }
}
if (bits) out += t[(acc << (6 - bits)) & 0x3f];
while (out.size() % 4) out += '=';
return out;
}

static std::string ppm(int w, int h, int maxval = 255, size_t trim = 0) {
std::string s = "P6\n" + std::to_string(w) + " " + std::to_string(h) + "\n" + std::to_string(maxval) + "\n";
std::string body(size_t(w) * size_t(h) * 3u, '\x7f');
for (size_t k = 0; k < body.size(); ++k) body[k] = char(k % 251);
if (trim && trim <= body.size()) body.resize(body.size() - trim);
return s + body;
}

int main() {
using onebit::npu::decode_image_data_url;
using onebit::npu::DecodedImage;

// 1. the happy path: a 2x1 P6 decodes to exactly its six samples
{
DecodedImage img;
const std::string p = ppm(2, 1);
check(decode_image_data_url("data:image/x-portable-pixmap;base64," + b64(p), img),
"a P6 PPM in a data URL decodes");
check(img.width == 2 && img.height == 1, "the geometry is the PPM's own");
check(img.rgb.size() == 6 && img.rgb[0] == uint8_t(p[p.size() - 6]),
"the samples are the PPM's bytes, in order");
}
// 2. an omitted media type is accepted (browsers emit `data:;base64,`)
{
DecodedImage img;
check(decode_image_data_url("data:;base64," + b64(ppm(1, 1)), img),
"an omitted media type is decoded as P6");
}
// 3. everything that is not a P6 is refused, by name
{
DecodedImage img;
check(!decode_image_data_url("data:image/png;base64," + b64("\x89PNG\r\n\x1a\n"), img),
"a PNG is refused");
check(img.error.find("image/png") != std::string::npos,
"...and the refusal names the media type it cannot decode");
}
{
DecodedImage img;
check(!decode_image_data_url("https://example.test/a.png", img), "a remote URL is refused");
check(img.error.find("does not fetch") != std::string::npos, "...and says it will not fetch");
}
// 4. malformed PPMs are refused rather than passed on as short buffers
{
DecodedImage img;
check(!decode_image_data_url("data:image/x-portable-pixmap;base64," + b64("P6\n2 1\n"), img),
"a PPM with no maxval is refused");
}
{
DecodedImage img;
check(!decode_image_data_url("data:image/x-portable-pixmap;base64," + b64(ppm(2, 2, 255, 5)), img),
"a PPM whose sample bytes are short is refused");
check(img.error.find("short") != std::string::npos, "...and says it is short");
}
{
DecodedImage img;
check(!decode_image_data_url("data:image/x-portable-pixmap;base64," + b64(ppm(1, 1, 16)), img),
"a PPM with maxval 16 is refused (two bytes per sample is a different format)");
}
{
DecodedImage img;
check(!decode_image_data_url("data:image/x-portable-pixmap;base64," + b64(ppm(0, 1)), img),
"a 0-wide PPM is refused");
}
{
DecodedImage img;
check(!decode_image_data_url("data:image/x-portable-pixmap;base64,!!!notbase64!!!", img),
"invalid base64 is refused");
}
{
DecodedImage img;
check(!decode_image_data_url("data:image/x-portable-pixmap," + ppm(1, 1), img),
"a non-base64 data URL is refused");
}
// 4b. the hostile header: a dimension past 2^31-1 is refused as out of range, not as a wrapped int. This is a parser of
// a REMOTE string, so the width is attacker-controlled; before the fix the multiply was signed-overflow UB and only the
// `w <= 0` check afterwards caught the wrap.
{
DecodedImage img;
check(!decode_image_data_url("data:image/x-portable-pixmap;base64," + b64("P6\n2147483648 2\n255\n"), img),
"a width past 2^31-1 is refused");
check(img.error.find("out of range") != std::string::npos,
"...and says the field is out of range rather than calling the header truncated");
}
{
DecodedImage img;
check(!decode_image_data_url("data:image/x-portable-pixmap;base64," + b64("P6\n99999999999999 2\n255\n"), img),
"a 14-digit width is refused too");
}
// 4c. trailing bytes ARE accepted, deliberately: a PPM stream may carry data after the samples. Pinned so the leniency
// cannot be mistaken for the "exact" the earlier comment claimed, and so the buffer handed downstream stays exactly w*h*3.
{
DecodedImage img;
const std::string p = ppm(2, 1) + "trailing";
check(decode_image_data_url("data:image/x-portable-pixmap;base64," + b64(p), img),
"a PPM with trailing bytes after the samples is still accepted");
check(img.rgb.size() == 6, "...and the buffer handed downstream is exactly w*h*3, not the trailing bytes too");
}

// 5. the decoder is not fooled by a P3 (ASCII PPM) header
{
DecodedImage img;
check(!decode_image_data_url("data:image/x-portable-pixmap;base64," + b64("P3\n1 1\n255\n0 0 0\n"), img),
"an ASCII PPM (P3) is refused: the chain wants binary samples");
}

if (g_fail) { std::printf("npu_image_part: %d of %d checks FAILED\n", g_fail, g_n); return 1; }
std::printf("npu_image_part: %d checks ok\n", g_n);
return 0;
}
Loading