diff --git a/app/npu_image_part.h b/app/npu_image_part.h new file mode 100644 index 000000000..81e1359c7 --- /dev/null +++ b/app/npu_image_part.h @@ -0,0 +1,166 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// npu_image_part.h -- turning an OpenAI `image_url` content part into the bytes the vision chain wants (engine#2324). +// +// The chain downstream of here is decoder-agnostic: npu/vision_image.h's preprocess() takes raw RGB plus its geometry, and +// npu/vision_tower.h's tower_forward() takes the same. What the serve route had no way to produce was those bytes, because an +// image part arrives as a data: URL and nothing in this tree decodes one. +// +// This decodes exactly one media type -- **image/x-portable-pixmap, P6, maxval 255** -- and refuses the rest *by name*, which is +// the same discipline as the route's refusal: a PNG handed to a route with no PNG decoder must not become "an image that produced +// nothing". P6 is the format npu/forward/minicpm_image_probe.cpp already reads, so a picture that works in the probe works here, +// and adding a real decoder later is a change to this one function. +// +// usage: +// DecodedImage img; +// if (!decode_image_data_url(url, img)) -> 400 with img.error (it names the media type) +// preprocess(img.rgb.data(), img.height, img.width, plan) + +#pragma once + +#include +#include +#include +#include + +namespace onebit::npu { + +struct DecodedImage { + std::vector rgb; // height * width * 3, row-major + int height = 0, width = 0; + std::string error; // set when the decode failed; names the media type when that was the reason +}; + +// RFC 4648 base64. Not a general-purpose encoder: the decoder is incremental-free and stops at the first bad character, which +// is what a request body needs -- anything after an invalid byte is not a picture. +inline bool base64_decode(const std::string& in, std::vector& out) { + static const int8_t kMap[256] = { + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,62,-1,-1,-1,63, 52,53,54,55,56,57,58,59,60,61,-1,-1,-1,-1,-1,-1, + -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14, 15,16,17,18,19,20,21,22,23,24,25,-1,-1,-1,-1,-1, + -1,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40, 41,42,43,44,45,46,47,48,49,50,51,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + }; + out.clear(); + uint32_t acc = 0; + int bits = 0; + for (unsigned char ch : in) { + if (ch == '=') break; // padding ends the stream + const int v = kMap[ch]; + if (v < 0) return false; + acc = (acc << 6) | uint32_t(v); + bits += 6; + if (bits >= 8) { + bits -= 8; + out.push_back(uint8_t((acc >> bits) & 0xffu)); + } + } + return true; +} + +// `data:image/x-portable-pixmap;base64,`, or `data:;base64,...` (the media type may be omitted, as browsers allow). +inline bool decode_image_data_url(const std::string& url, DecodedImage& out) { + out = DecodedImage{}; + if (url.rfind("data:", 0) != 0) { + out.error = "the image part is not a data: URL; this route does not fetch remote images (engine#2324)"; + return false; + } + const size_t comma = url.find(','); + if (comma == std::string::npos) { + out.error = "the image part has no ',' after its data: header"; + return false; + } + const std::string header = url.substr(5, comma - 5); + const bool b64 = header.find(";base64") != std::string::npos; + if (!b64) { + out.error = "the image part is a data: URL that is not base64-encoded; only ;base64 is accepted (engine#2324)"; + return false; + } + std::string media = header.substr(0, header.find(';')); + if (media.empty()) media = "image/x-portable-pixmap"; // `data:;base64,` means "you decide" + if (media != "image/x-portable-pixmap" && media != "image/x-ppm" && media != "image/ppm") { + out.error = "no decoder for media type '" + media + "': this route decodes image/x-portable-pixmap (P6) only, and " + "refuses the rest by name rather than answering without the picture (engine#2324)"; + return false; + } + std::vector raw; + if (!base64_decode(url.substr(comma + 1), raw)) { + out.error = "the image part's base64 payload is not valid base64"; + return false; + } + // P6: "P6" whitespace W H whitespace 255 single-whitespace then binary RGB. Comments ('#') may appear in the header. + size_t i = 0; + auto skip_ws_and_comments = [&]() { + while (i < raw.size()) { + if (raw[i] == '#') { while (i < raw.size() && raw[i] != '\n') ++i; } + else if (raw[i] == ' ' || raw[i] == '\t' || raw[i] == '\r' || raw[i] == '\n') ++i; + else break; + } + }; + // Accumulated in int64_t and bounded BEFORE it can overflow an int: the input is a remote data: URL, so a header + // claiming a width of 10^14 is a hostile value, not a typo. `int` arithmetic here was undefined behaviour that happened + // to be caught afterwards by the `w <= 0` check -- a parser must not rely on the wrap landing somewhere detectable. + bool range_error = false; + auto read_uint = [&](int& v) { + skip_ws_and_comments(); + if (i >= raw.size() || raw[i] < '0' || raw[i] > '9') return false; + int64_t acc = 0; + while (i < raw.size() && raw[i] >= '0' && raw[i] <= '9') { + acc = acc * 10 + (raw[i] - '0'); + ++i; + if (acc > 2147483647LL) { range_error = true; return false; } + } + v = int(acc); + return true; + }; + if (raw.size() < 2 || raw[0] != 'P' || raw[1] != '6') { + out.error = "the decoded image is not a P6 PPM ('P6' magic missing)"; + return false; + } + i = 2; + int h = 0, w = 0, maxval = 0; + if (!read_uint(w) || !read_uint(h) || !read_uint(maxval)) { + out.error = range_error ? "a PPM header field is out of range (a dimension is larger than 2^31-1)" + : "the PPM header is truncated (width, height and maxval are required)"; + return false; + } + if (maxval != 255) { + out.error = "the PPM has maxval " + std::to_string(maxval) + "; only 255 (one byte per sample) is accepted"; + return false; + } + if (w <= 0 || h <= 0) { + out.error = "the PPM is " + std::to_string(w) + "x" + std::to_string(h) + ", which is not an image"; + return false; + } + ++i; // exactly one whitespace byte after maxval, then binary samples + // "At least", not "exactly": a PPM stream may carry trailing data after the samples, and refusing it would reject files + // that are correct. What the buffer handed downstream is -- exactly w*h*3 bytes -- is what has to be exact, and that is + // the `assign` below rather than this test. + const size_t need = size_t(w) * size_t(h) * 3u; + if (raw.size() < i + need) { + out.error = "the PPM is short: " + std::to_string(raw.size() - i) + " of " + std::to_string(need) + " sample bytes"; + return false; + } + out.rgb.assign(raw.begin() + long(i), raw.begin() + long(i + need)); + out.height = h; + out.width = w; + return true; +} + +} // namespace onebit::npu diff --git a/cmake/tests-image-part.cmake b/cmake/tests-image-part.cmake new file mode 100644 index 000000000..cd519267a --- /dev/null +++ b/cmake/tests-image-part.cmake @@ -0,0 +1,21 @@ +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# engine#2324: the one place that knows what a picture arrives as -- a data: URL -- turning it into the raw RGB plus geometry +# that npu/vision_image.h and npu/vision_tower.h already take. Header-only, no model, no device. +add_executable(npu_image_part_test tests/npu_image_part_test.cpp) +target_include_directories(npu_image_part_test PRIVATE app) +add_test(NAME npu_image_part COMMAND npu_image_part_test) +add_dependencies(onebit npu_image_part_test) diff --git a/tests/npu_image_part_test.cpp b/tests/npu_image_part_test.cpp new file mode 100644 index 000000000..5555baf34 --- /dev/null +++ b/tests/npu_image_part_test.cpp @@ -0,0 +1,155 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// npu_image_part_test.cpp -- the data: URL decode in front of the vision chain (engine#2324). +// +// Downstream of this function the chain is decoder-agnostic (npu/vision_image.h takes raw RGB plus geometry), so this is the one +// place that knows what a picture arrives as. Both directions are pinned: a P6 PPM decodes to exactly its samples, and every way +// of not being one -- a PNG, a remote URL, a truncated header, a short payload, maxval 16, a 0x0 image -- is refused with a +// message, because "an image that produced nothing" is the failure this file exists to prevent. +#include +#include +#include + +#include "npu_image_part.h" + +static int g_fail = 0, g_n = 0; +static void check(bool ok, const std::string& what) { + ++g_n; + if (!ok) { std::printf("FAIL %s\n", what.c_str()); ++g_fail; } +} + +static std::string b64(const std::string& raw) { + static const char* t = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + std::string out; + uint32_t acc = 0; int bits = 0; + for (unsigned char c : raw) { + acc = (acc << 8) | c; bits += 8; + while (bits >= 6) { bits -= 6; out += t[(acc >> bits) & 0x3f]; } + } + if (bits) out += t[(acc << (6 - bits)) & 0x3f]; + while (out.size() % 4) out += '='; + return out; +} + +static std::string ppm(int w, int h, int maxval = 255, size_t trim = 0) { + std::string s = "P6\n" + std::to_string(w) + " " + std::to_string(h) + "\n" + std::to_string(maxval) + "\n"; + std::string body(size_t(w) * size_t(h) * 3u, '\x7f'); + for (size_t k = 0; k < body.size(); ++k) body[k] = char(k % 251); + if (trim && trim <= body.size()) body.resize(body.size() - trim); + return s + body; +} + +int main() { + using onebit::npu::decode_image_data_url; + using onebit::npu::DecodedImage; + + // 1. the happy path: a 2x1 P6 decodes to exactly its six samples + { + DecodedImage img; + const std::string p = ppm(2, 1); + check(decode_image_data_url("data:image/x-portable-pixmap;base64," + b64(p), img), + "a P6 PPM in a data URL decodes"); + check(img.width == 2 && img.height == 1, "the geometry is the PPM's own"); + check(img.rgb.size() == 6 && img.rgb[0] == uint8_t(p[p.size() - 6]), + "the samples are the PPM's bytes, in order"); + } + // 2. an omitted media type is accepted (browsers emit `data:;base64,`) + { + DecodedImage img; + check(decode_image_data_url("data:;base64," + b64(ppm(1, 1)), img), + "an omitted media type is decoded as P6"); + } + // 3. everything that is not a P6 is refused, by name + { + DecodedImage img; + check(!decode_image_data_url("data:image/png;base64," + b64("\x89PNG\r\n\x1a\n"), img), + "a PNG is refused"); + check(img.error.find("image/png") != std::string::npos, + "...and the refusal names the media type it cannot decode"); + } + { + DecodedImage img; + check(!decode_image_data_url("https://example.test/a.png", img), "a remote URL is refused"); + check(img.error.find("does not fetch") != std::string::npos, "...and says it will not fetch"); + } + // 4. malformed PPMs are refused rather than passed on as short buffers + { + DecodedImage img; + check(!decode_image_data_url("data:image/x-portable-pixmap;base64," + b64("P6\n2 1\n"), img), + "a PPM with no maxval is refused"); + } + { + DecodedImage img; + check(!decode_image_data_url("data:image/x-portable-pixmap;base64," + b64(ppm(2, 2, 255, 5)), img), + "a PPM whose sample bytes are short is refused"); + check(img.error.find("short") != std::string::npos, "...and says it is short"); + } + { + DecodedImage img; + check(!decode_image_data_url("data:image/x-portable-pixmap;base64," + b64(ppm(1, 1, 16)), img), + "a PPM with maxval 16 is refused (two bytes per sample is a different format)"); + } + { + DecodedImage img; + check(!decode_image_data_url("data:image/x-portable-pixmap;base64," + b64(ppm(0, 1)), img), + "a 0-wide PPM is refused"); + } + { + DecodedImage img; + check(!decode_image_data_url("data:image/x-portable-pixmap;base64,!!!notbase64!!!", img), + "invalid base64 is refused"); + } + { + DecodedImage img; + check(!decode_image_data_url("data:image/x-portable-pixmap," + ppm(1, 1), img), + "a non-base64 data URL is refused"); + } + // 4b. the hostile header: a dimension past 2^31-1 is refused as out of range, not as a wrapped int. This is a parser of + // a REMOTE string, so the width is attacker-controlled; before the fix the multiply was signed-overflow UB and only the + // `w <= 0` check afterwards caught the wrap. + { + DecodedImage img; + check(!decode_image_data_url("data:image/x-portable-pixmap;base64," + b64("P6\n2147483648 2\n255\n"), img), + "a width past 2^31-1 is refused"); + check(img.error.find("out of range") != std::string::npos, + "...and says the field is out of range rather than calling the header truncated"); + } + { + DecodedImage img; + check(!decode_image_data_url("data:image/x-portable-pixmap;base64," + b64("P6\n99999999999999 2\n255\n"), img), + "a 14-digit width is refused too"); + } + // 4c. trailing bytes ARE accepted, deliberately: a PPM stream may carry data after the samples. Pinned so the leniency + // cannot be mistaken for the "exact" the earlier comment claimed, and so the buffer handed downstream stays exactly w*h*3. + { + DecodedImage img; + const std::string p = ppm(2, 1) + "trailing"; + check(decode_image_data_url("data:image/x-portable-pixmap;base64," + b64(p), img), + "a PPM with trailing bytes after the samples is still accepted"); + check(img.rgb.size() == 6, "...and the buffer handed downstream is exactly w*h*3, not the trailing bytes too"); + } + + // 5. the decoder is not fooled by a P3 (ASCII PPM) header + { + DecodedImage img; + check(!decode_image_data_url("data:image/x-portable-pixmap;base64," + b64("P3\n1 1\n255\n0 0 0\n"), img), + "an ASCII PPM (P3) is refused: the chain wants binary samples"); + } + + if (g_fail) { std::printf("npu_image_part: %d of %d checks FAILED\n", g_fail, g_n); return 1; } + std::printf("npu_image_part: %d checks ok\n", g_n); + return 0; +}