Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions app/forward_serve.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -270,6 +270,18 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib
// The functions the request offers go the same way (npu::request_template_vars): the
// template is where a model learns what it may call.
const nlohmann::ordered_json extra = npu::request_template_vars(ordered, npu::thinking_from_body(body));
// engine#2324: this route has no vision wiring, so a picture in the request is refused by name rather
// than rendered as text and answered. The chain it would call exists; the parse and the join here do not.
if (npu::message_carries_an_image(ordered.at("messages"))) {
res.status = 400;
res.set_content(nlohmann::ordered_json{{"error", nlohmann::ordered_json{{"message",
"this route has no vision wiring yet (engine#2324): the request carries an image part and the "
"NPU route parses no pixels, so answering it would ignore the picture. Text-only requests are "
"unaffected. The vision chain exists (npu/vision_tower.h and friends) and is driven by "
"npu/forward/minicpm_image_probe.cpp; the serve-side parse and the image positions are what is missing."},
{"type", "unsupported_image_part"}, {"code", 400}}}.dump(), "application/json");
return;
}
// An earlier call's arguments arrive as a JSON string; the template is written against the
// object (npu::messages_for_template).
prompt = npu::render_chat_template(e.tmpl, npu::messages_for_template(ordered.at("messages")),
Expand Down
21 changes: 21 additions & 0 deletions app/npu_template.h
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,27 @@ inline nlohmann::ordered_json messages_for_template(nlohmann::ordered_json messa
return messages;
}

// Does any message carry a PICTURE? (engine#2324)
//
// OpenAI spells an image as a content-part array: `content: [{"type":"text",...}, {"type":"image_url","image_url":{...}}]`.
// This route parses no pixels -- there is no base64 decode and no call into npu/vision_*.h -- so a request that carries one
// would otherwise reach the template as an array and be answered as if the picture were text: a wrong answer with a 200.
// The vision chain itself exists (npu/vision_tower.h, vision_prompt.h, vision_plan.h, vision_image.h, driven by
// npu/forward/minicpm_image_probe.cpp); what is missing is the serve wiring, so this refuses rather than guesses.
inline bool message_carries_an_image(const nlohmann::ordered_json& messages) {
if (!messages.is_array()) return false;
for (const auto& m : messages) {
if (!m.is_object() || !m.contains("content") || !m["content"].is_array()) continue;
for (const auto& part : m["content"]) {
if (!part.is_object() || !part.contains("type") || !part["type"].is_string()) continue;
const std::string type = part["type"].get<std::string>();
// `image_url` is the OpenAI spelling; `image` is what several local servers accept. Both mean pixels.
if (type == "image_url" || type == "image") return true;
}
}
return false;
}

// Render `messages` with the container's template.
//
// messages the request's messages array, as the OpenAI API spells it
Expand Down
26 changes: 26 additions & 0 deletions tests/npu_template_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,32 @@ int main(int argc, char** argv) {
check(onebit::npu::messages_for_template(odd) == odd, "call arguments: a string that is not a JSON object is left as sent");
}

// engine#2324: which requests carry a PICTURE. The NPU route parses no pixels, so the route refuses by name instead of
// rendering the part as text and answering; these arms are the predicate that decides, and both directions matter --
// over-matching would break the tool-call requests above, under-matching would answer a picture as if it were text.
{
using onebit::npu::message_carries_an_image;
const auto m = [](const char* text) { return nlohmann::ordered_json::parse(text); };
check(!message_carries_an_image(m(R"([{"role":"user","content":"hello"}])")),
"an image part: a plain string content is not a picture");
check(!message_carries_an_image(m(R"([{"role":"user","content":[{"type":"text","text":"hello"}]}])")),
"an image part: a text-only content ARRAY is not a picture");
check(message_carries_an_image(
m(R"([{"role":"user","content":[{"type":"text","text":"what is this?"},{"type":"image_url","image_url":{"url":"data:image/ppm;base64,AAA"}}]}])")),
"an image part: the OpenAI image_url part is a picture");
check(message_carries_an_image(m(R"([{"role":"user","content":[{"type":"image","data":"AAA"}]}])")),
"an image part: the bare image spelling several local servers accept is a picture too");
check(message_carries_an_image(
m(R"([{"role":"system","content":"x"},{"role":"user","content":[{"type":"image_url","image_url":{"url":"http://h/i.png"}}]}])")),
"an image part: it is found in a later message, not only the first");
check(!message_carries_an_image(m(R"({"not":"an array"})")),
"an image part: a body whose messages are not an array is not a picture");
check(!message_carries_an_image(m(R"([{"role":"user","content":[{"no_type":1},{"type":2}]}])")),
"an image part: parts without a string type are not pictures");
check(!message_carries_an_image(m(R"([{"role":"assistant","tool_calls":[{"function":{"arguments":"{\"type\":\"image_url\"}"}}]}])")),
"an image part: the string image_url inside tool arguments is not a picture (it is not a content part)");
}

// A container that declares no template is this layer's "no", not a ChatML render: the route decides
// what to do about that (app/forward_serve.cpp: ChatML with anchors, else the #987 400).
{
Expand Down
Loading