From 6643451b8009ccf235f4a0561734a37701f2d3ff Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 12 Nov 2026 23:03:33 -0400 Subject: [PATCH] serve: refuse a request that carries an image, instead of answering as if the picture were text (engine#2324) The NPU route parses no pixels: `app/` has zero `image_url`, `data:image` or base64 content parts, and nothing calls the vision chain. But `messages_for_template()` passes a content-part ARRAY straight to the chat template, so a request with an image reached the template and was answered -- a wrong answer with a 200, about a picture the route never read. It is now refused by name, before rendering: {"error": {"message": "this route has no vision wiring yet (engine#2324): the request carries an image part and the NPU route parses no pixels, so answering it would ignore the picture. Text-only requests are unaffected. The vision chain exists (npu/vision_tower.h and friends) and is driven by npu/forward/minicpm_image_probe.cpp; the serve-side parse and the image positions are what is missing.", "type": "unsupported_image_part", "code": 400}} Two spellings count as a picture (`image_url`, the OpenAI form, and the bare `image` several local servers accept), and the predicate deliberately does NOT match a `"type":"image_url"` string that appears inside a tool call's JSON arguments -- that is text the template is entitled to see, and matching it would break the tool-call requests the same test file already covers. Eight arms in `tests/npu_template_test.cpp`, both directions: a plain string content, a text-only array, a body whose messages are not an array, parts without a string type, and the tool-arguments string are NOT pictures; an `image_url` part, a bare `image` part, and one in a later message ARE. Built and run here: all eight pass. This is (6)'s precondition, not (6). The vision chain is on main -- `npu/vision_tower.{h,cpp}` (`tower_forward(pack, pixels, gh, gw, tower*)`), `vision_plan.h`, `vision_prompt.h` (`ImageMarkers`, `image_tokens`), `vision_image.{h,cpp}`, `vision_pack.{h,cpp}` -- and `npu/forward/minicpm_image_probe.cpp` drives all of it on the host. What is missing is the serve-side parse and the join, and (8)'s cache key: `npu_prefix_cache.h` matches a slot on token ids alone, so two different pictures at the same placeholder positions would reuse the same KV state. --- app/forward_serve.cpp | 12 ++++++++++++ app/npu_template.h | 21 +++++++++++++++++++++ tests/npu_template_test.cpp | 26 ++++++++++++++++++++++++++ 3 files changed, 59 insertions(+) diff --git a/app/forward_serve.cpp b/app/forward_serve.cpp index 62c419cc4..3724e313d 100644 --- a/app/forward_serve.cpp +++ b/app/forward_serve.cpp @@ -270,6 +270,18 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib // The functions the request offers go the same way (npu::request_template_vars): the // template is where a model learns what it may call. const nlohmann::ordered_json extra = npu::request_template_vars(ordered, npu::thinking_from_body(body)); + // engine#2324: this route has no vision wiring, so a picture in the request is refused by name rather + // than rendered as text and answered. The chain it would call exists; the parse and the join here do not. + if (npu::message_carries_an_image(ordered.at("messages"))) { + res.status = 400; + res.set_content(nlohmann::ordered_json{{"error", nlohmann::ordered_json{{"message", + "this route has no vision wiring yet (engine#2324): the request carries an image part and the " + "NPU route parses no pixels, so answering it would ignore the picture. Text-only requests are " + "unaffected. The vision chain exists (npu/vision_tower.h and friends) and is driven by " + "npu/forward/minicpm_image_probe.cpp; the serve-side parse and the image positions are what is missing."}, + {"type", "unsupported_image_part"}, {"code", 400}}}.dump(), "application/json"); + return; + } // An earlier call's arguments arrive as a JSON string; the template is written against the // object (npu::messages_for_template). prompt = npu::render_chat_template(e.tmpl, npu::messages_for_template(ordered.at("messages")), diff --git a/app/npu_template.h b/app/npu_template.h index 0137e85cc..b6b5548b8 100644 --- a/app/npu_template.h +++ b/app/npu_template.h @@ -142,6 +142,27 @@ inline nlohmann::ordered_json messages_for_template(nlohmann::ordered_json messa return messages; } +// Does any message carry a PICTURE? (engine#2324) +// +// OpenAI spells an image as a content-part array: `content: [{"type":"text",...}, {"type":"image_url","image_url":{...}}]`. +// This route parses no pixels -- there is no base64 decode and no call into npu/vision_*.h -- so a request that carries one +// would otherwise reach the template as an array and be answered as if the picture were text: a wrong answer with a 200. +// The vision chain itself exists (npu/vision_tower.h, vision_prompt.h, vision_plan.h, vision_image.h, driven by +// npu/forward/minicpm_image_probe.cpp); what is missing is the serve wiring, so this refuses rather than guesses. +inline bool message_carries_an_image(const nlohmann::ordered_json& messages) { + if (!messages.is_array()) return false; + for (const auto& m : messages) { + if (!m.is_object() || !m.contains("content") || !m["content"].is_array()) continue; + for (const auto& part : m["content"]) { + if (!part.is_object() || !part.contains("type") || !part["type"].is_string()) continue; + const std::string type = part["type"].get(); + // `image_url` is the OpenAI spelling; `image` is what several local servers accept. Both mean pixels. + if (type == "image_url" || type == "image") return true; + } + } + return false; +} + // Render `messages` with the container's template. // // messages the request's messages array, as the OpenAI API spells it diff --git a/tests/npu_template_test.cpp b/tests/npu_template_test.cpp index 9ba43d17c..e8fae2d53 100644 --- a/tests/npu_template_test.cpp +++ b/tests/npu_template_test.cpp @@ -178,6 +178,32 @@ int main(int argc, char** argv) { check(onebit::npu::messages_for_template(odd) == odd, "call arguments: a string that is not a JSON object is left as sent"); } + // engine#2324: which requests carry a PICTURE. The NPU route parses no pixels, so the route refuses by name instead of + // rendering the part as text and answering; these arms are the predicate that decides, and both directions matter -- + // over-matching would break the tool-call requests above, under-matching would answer a picture as if it were text. + { + using onebit::npu::message_carries_an_image; + const auto m = [](const char* text) { return nlohmann::ordered_json::parse(text); }; + check(!message_carries_an_image(m(R"([{"role":"user","content":"hello"}])")), + "an image part: a plain string content is not a picture"); + check(!message_carries_an_image(m(R"([{"role":"user","content":[{"type":"text","text":"hello"}]}])")), + "an image part: a text-only content ARRAY is not a picture"); + check(message_carries_an_image( + m(R"([{"role":"user","content":[{"type":"text","text":"what is this?"},{"type":"image_url","image_url":{"url":"data:image/ppm;base64,AAA"}}]}])")), + "an image part: the OpenAI image_url part is a picture"); + check(message_carries_an_image(m(R"([{"role":"user","content":[{"type":"image","data":"AAA"}]}])")), + "an image part: the bare image spelling several local servers accept is a picture too"); + check(message_carries_an_image( + m(R"([{"role":"system","content":"x"},{"role":"user","content":[{"type":"image_url","image_url":{"url":"http://h/i.png"}}]}])")), + "an image part: it is found in a later message, not only the first"); + check(!message_carries_an_image(m(R"({"not":"an array"})")), + "an image part: a body whose messages are not an array is not a picture"); + check(!message_carries_an_image(m(R"([{"role":"user","content":[{"no_type":1},{"type":2}]}])")), + "an image part: parts without a string type are not pictures"); + check(!message_carries_an_image(m(R"([{"role":"assistant","tool_calls":[{"function":{"arguments":"{\"type\":\"image_url\"}"}}]}])")), + "an image part: the string image_url inside tool arguments is not a picture (it is not a content part)"); + } + // A container that declares no template is this layer's "no", not a ChatML render: the route decides // what to do about that (app/forward_serve.cpp: ChatML with anchors, else the #987 400). {