From e7cb1ebab02e1775926023ac870bdd8b29157678 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Fri, 13 Nov 2026 00:57:24 -0400 Subject: [PATCH 1/3] forward-serve: pictures on the chat route, with a prompt cache keyed on the picture (refs #2324) A pack with a vision component (MiniCPM-V 4.7: vision_weight.q4nx) now takes OpenAI image_url parts on /v1/chat/completions: - data: URLs only; PNG, JPEG and binary PPM decoded by stb_image (pinned via FetchContent, built with those three formats only). Over-large pictures, a part with no URL and a pack with no vision component are named 400s. - npu/vision_request.h replaces each <|image_pad|> the pack's template renders with the picture's tokens, runs the tower per piece on the host, and lays out the three position planes; the prompt goes through step_batch_embeds in the route's chunks, and decoding resumes at the planes' next position. - the prompt cache is keyed on the ids with each picture row replaced by a key from that picture's bytes: two pictures of one size expand to identical ids. The ids, positions and key come from the plan alone, the lookup runs first, and the tower runs only when a picture row lies past the resume point. - speculative decoding is off for a request with pictures. Measured, MiniCPM-V-4.7-1B-NPU2 on the NPU, a 672 x 1344 picture: red 43.0 s, the same picture again 1.1 s (485 of 486 rows cached, no tower), blue at the same size 55.5 s and answered "Blue". Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 18 ++++- app/forward_serve.cpp | 129 +++++++++++++++++++++++++---- app/npu_image_request.h | 125 ++++++++++++++++++++++++++++ app/stb_image_impl.cpp | 26 ++++++ docs/npu.md | 49 ++++++++++- npu/vision_request.cpp | 134 +++++++++++++++++++++++++++++++ npu/vision_request.h | 74 +++++++++++++++++ tests/npu_image_request_test.cpp | 118 +++++++++++++++++++++++++++ 8 files changed, 653 insertions(+), 20 deletions(-) create mode 100644 app/npu_image_request.h create mode 100644 app/stb_image_impl.cpp create mode 100644 npu/vision_request.cpp create mode 100644 npu/vision_request.h create mode 100644 tests/npu_image_request_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index b81dca211..fdab08195 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -85,7 +85,12 @@ set(HTTPLIB_INSTALL OFF CACHE BOOL "" FORCE) FetchContent_Declare(httplib GIT_REPOSITORY https://github.com/yhirose/cpp-httplib.git GIT_TAG fe332fa06bac76a1c6d402c08f414052999347da) # v0.47.0 -FetchContent_MakeAvailable(json httplib) +# engine#2324: images in a chat request arrive as PNG or JPEG bytes; stb_image decodes them (one header, public domain / +# MIT). It has no CMakeLists, so it is only fetched; app/stb_image_impl.cpp compiles it with PNG, JPEG and PNM only. +FetchContent_Declare(stb + GIT_REPOSITORY https://github.com/nothings/stb.git + GIT_TAG 9600522de53973336fa033a289858a4354cbb04c) +FetchContent_MakeAvailable(json httplib stb) # config/laya-first.json (the routing table, docs/laya-first.md §4) is compiled in like # config/recipes.json; `1bit laya-first --laya-first FILE` replaces it at run time. @@ -903,7 +908,7 @@ option(ONEBIT_NPU_HOST "Build the NPU host library and its tests" ${_onebit_npu_ if(ONEBIT_NPU_HOST) find_package(PkgConfig REQUIRED) pkg_check_modules(PCRE2 REQUIRED IMPORTED_TARGET libpcre2-8) -add_library(onebit_npu STATIC npu/file_map.cpp npu/flm_head_import.cpp npu/full_elf.cpp npu/model.cpp npu/vision_pack.cpp npu/vision_tower.cpp npu/vision_image.cpp +add_library(onebit_npu STATIC npu/file_map.cpp npu/flm_head_import.cpp npu/full_elf.cpp npu/model.cpp npu/vision_pack.cpp npu/vision_tower.cpp npu/vision_image.cpp npu/vision_request.cpp npu/audio_exec.cpp npu/moe.cpp npu/pack.cpp npu/pack_import.cpp npu/private_route.cpp npu/q4nx.cpp npu/q4nx_device.cpp npu/tokenizer.cpp) target_include_directories(onebit_npu PUBLIC npu) @@ -1452,6 +1457,12 @@ add_executable(npu_vision_image_test tests/npu_vision_image_test.cpp) target_link_libraries(npu_vision_image_test PRIVATE onebit_npu) add_test(NAME npu_vision_image COMMAND npu_vision_image_test) add_dependencies(onebit npu_vision_image_test) +# engine#2324: a chat request's pictures (data URLs, PNG / JPEG / PPM) to RGB bytes, exactly. +add_executable(npu_image_request_test tests/npu_image_request_test.cpp app/stb_image_impl.cpp) +target_include_directories(npu_image_request_test PRIVATE app npu ${stb_SOURCE_DIR}) +target_link_libraries(npu_image_request_test PRIVATE onebit_npu nlohmann_json::nlohmann_json) +add_test(NAME npu_image_request COMMAND npu_image_request_test) +add_dependencies(onebit npu_image_request_test) # The text side of an image: its tokens and the canvas positions, against the reference model code's own answers. add_executable(npu_vision_prompt_test tests/npu_vision_prompt_test.cpp) target_include_directories(npu_vision_prompt_test PRIVATE npu) @@ -2105,7 +2116,8 @@ endif() # The library body is onebit_npu_forward_library() (defined above the runtime options), so the CI build of the # same sources against the runtime stub (engine#1946) cannot drift from this one. onebit_npu_forward_library(onebit_npu_forward onebit_npu_rt) - target_sources(onebit PRIVATE app/forward_serve.cpp) + target_sources(onebit PRIVATE app/forward_serve.cpp app/stb_image_impl.cpp) # engine#2324: stb_image for pictures + target_include_directories(onebit PRIVATE ${stb_SOURCE_DIR}) # engine#1781: forward-serve --spec-draft runs a DFlash draft package (npu/dflash_head.h) as its drafter. target_sources(onebit PRIVATE npu/dflash_forward.cpp npu/dflash_draft.cpp) # engine#2111: the usage object's pure function, its own TU so tests/serve_usage_test.cpp can link it without diff --git a/app/forward_serve.cpp b/app/forward_serve.cpp index 62c419cc4..75defd681 100644 --- a/app/forward_serve.cpp +++ b/app/forward_serve.cpp @@ -55,6 +55,8 @@ #include "stream_bridge.h" #include "npu_systemone.h" #include "tokenizer.h" +#include "npu_image_request.h" // engine#2324: a chat request's pictures +#include "vision_pack.h" #include "../npu/decision.h" // the d3 readout head (npu/decision.h, issue #653) #include "../npu/mtp/prompt_lookup.h" // --spec-lookup: the speculative loop with a prompt-lookup drafter (#1781) #include "../npu/dflash_head.h" // --spec-draft: the same loop with a DFlash draft package (#1781) @@ -99,6 +101,9 @@ struct ForwardEngine { std::string id; int lock_fd = -1; // app/npu_device_lock.h: the device lock, held for the process's life std::unique_ptr tok; + // engine#2324: the pack's vision component (vision_weight.q4nx), when it has one: a chat request with pictures is + // served through it (npu/vision_request.h); without one such a request is a 400. + std::unique_ptr vision; int bos_id = -1; // config.json bos_token_id (the chat template may open with it) bool add_bos = false; // config.json add_bos_token: the rendered chat prompt needs BOS std::unique_ptr fwd; @@ -257,6 +262,8 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib // and can run a few tokens into the question when two questions open with the same words. Empty = the // request has no system message, or the route is not pinning. std::string system_prompt; + // engine#2324: the request's pictures, decoded, in the order the template renders their pads. + auto images = std::make_shared>(); try { if (chat) { if (!e.chat_ok) throw std::runtime_error(e.chat_refusal); @@ -267,6 +274,8 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib // template that serializes its input (a tools block through `tojson`) renders the // request's key order into the prompt, so it has to be the caller's order, not ours. const nlohmann::ordered_json ordered = nlohmann::ordered_json::parse(req.body); + if (npu::request_images(ordered.at("messages"), *images) && !e.vision) + throw std::runtime_error("this model has no vision component, so it takes no pictures"); // The functions the request offers go the same way (npu::request_template_vars): the // template is where a model learns what it may call. const nlohmann::ordered_json extra = npu::request_template_vars(ordered, npu::thinking_from_body(body)); @@ -288,6 +297,8 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib } } } else { + if (npu::request_images(body.at("messages"), *images)) + throw std::runtime_error("pictures need the pack's own chat template, and this build or pack has none"); prompt = npu::chatml(body.at("messages"), npu::thinking_from_body(body)); if (e.prefix_pin) { const json probe = npu::pin_probe_messages(body.at("messages")); @@ -350,7 +361,7 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib // Returns the finish_reason the request earned: "stop" for an end-of-sequence token, // a stop sequence or a grammar that allows nothing more, "length" when the token // budget ran out. - auto run = [&e, &req, prompt, system_prompt, prompt_ids, prompt_is_ids, max_tokens, chat, &opts, gram, &stats, dump_req, bridge]( + auto run = [&e, &req, prompt, system_prompt, prompt_ids, prompt_is_ids, max_tokens, chat, &opts, gram, &stats, dump_req, bridge, images]( const std::function&)>& on_text) { std::string finish = "stop"; // The tokenizer is const and keeps no per-call state, so it runs before the lock. A caller @@ -366,6 +377,35 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib if (!prompt_is_ids && chat && e.add_bos && e.bos_id >= 0 && (ids.empty() || ids.front() != e.bos_id)) ids.insert(ids.begin(), e.bos_id); if (ids.empty()) throw std::runtime_error("empty prompt"); + // engine#2324: pictures. Each pad the template rendered becomes the picture's own tokens; the tower's rows go + // where its placeholders are and the embedding everywhere else (npu/vision_request.h). The prompt then runs + // through step_batch_embeds with three position planes, and decoding resumes at `img_next` on those planes + // while the cache slot keeps counting from the prompt's length. The tower runs here, before the lock. + const bool img = !images->empty(); + npu::vision::ExpandedPrompt ex; + std::vector img_embeds; // n x H, the whole prompt's rows + std::vector key_ids; // what the prompt cache is keyed on: ids, with each picture's rows made its own + int img_next = 0; + if (img) { + std::string why; + // The tower is not run here: a prompt-cache hit that covers every image row does not need it. + if (!npu::vision::expand_images(*e.tok, *e.vision, ids, *images, ex, why, 4096, /*run_tower=*/false)) + throw std::runtime_error("pictures: " + why); + ids = ex.ids; + img_next = ex.positions.next; + key_ids = ids; + size_t row = 0; + for (size_t i = 0; i < ids.size(); i++) + if (ids[i] == ex.image_pad) { + // A key id no token has (negative), and a different one for a different picture: two pictures of one + // size expand to identical ids, and an ids-only key would hand one picture's state to the other. + key_ids[i] = -int(1 + ((ex.image_hash ^ (uint64_t(row) * 0x9E3779B97F4A7C15ull)) & 0x3FFFFFFF)); + ++row; + } + std::fprintf(stderr, "[image] %d picture(s): %zu prompt rows, %zu of them image rows; decoding resumes at position %d\n", + ex.images, ids.size(), row, img_next); + } + const std::vector& cache_ids = img ? key_ids : ids; stats.prompt_tokens = int(ids.size()); // The KV cache grows with every position: the prompt and the answer together stay // inside the model's context. @@ -389,7 +429,7 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib // state, not logits, so that token is run again. int resume = 0; if (e.prefix && e.prefix->enabled()) { - const npu::PrefixDecision d = e.prefix->lookup(ids); + const npu::PrefixDecision d = e.prefix->lookup(cache_ids); stats.prefix = d; // One token short of both the slot and the prompt: the snapshot stops one token short (see the save // below), and the last token is run again for the logits the answer is drawn from. @@ -410,6 +450,38 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib std::fprintf(stderr, "[prefix] slot=%d cached=%d of %zu reason=%s\n", stats.prefix.slot, stats.prefix.cached_tokens, ids.size(), stats.prefix.reason.c_str()); } + if (img) { + // Rows from `resume` on are what the prefill reads. The tower runs only when one of them is a picture's: a + // cache hit past every picture row answers without it (engine#2324, condition 8). Under the lock, as + // embed_row() reads through the forward's one-row cache. + bool need_tower = false; + for (size_t i = size_t(resume); i < ids.size() && !need_tower; i++) need_tower = ids[i] == ex.image_pad; + if (need_tower) { + std::string why; + if (!npu::vision::tower_rows(*e.vision, *images, ex, why)) throw std::runtime_error("pictures: " + why); + } + std::vector one; + e.fwd->embed_row(ids[0], one); + const size_t H = one.size(); + if (size_t(ex.width) != H) + throw std::runtime_error("pictures: the tower projects to " + std::to_string(ex.width) + " but the text model is " + + std::to_string(H) + " wide"); + img_embeds.assign(ids.size() * H, 0.0f); // rows before `resume` come from the cache and are never read + size_t row = 0; + for (size_t i = 0; i < ids.size(); i++) { + const bool pad = ids[i] == ex.image_pad; + if (i >= size_t(resume)) { + if (pad) std::copy(ex.rows.begin() + long(row * H), ex.rows.begin() + long((row + 1) * H), img_embeds.begin() + long(i * H)); + else { + e.fwd->embed_row(ids[i], one); + std::copy(one.begin(), one.end(), img_embeds.begin() + long(i * H)); + } + } + row += pad; + } + std::fprintf(stderr, "[image] tower %s (prefill from row %d of %zu)\n", need_tower ? "run" : "not needed: the cache covers every picture row", + resume, ids.size()); + } // (engine#1785) Where this request should leave a pinned snapshot, as a position: the state is saved // one token short of the shared part for the same reason the whole-prompt snapshot below is saved one // token short of the prompt. -1 = no pin for this request. pin_point() is 0 when a slot (the pin @@ -424,12 +496,29 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib std::vector lead = e.tok->encode(system_prompt); if (chat && e.add_bos && e.bos_id >= 0 && (lead.empty() || lead.front() != e.bos_id)) lead.insert(lead.begin(), e.bos_id); - pin_tokens = e.prefix->pin_at(ids, npu::shared_prefix_len(lead, ids)); + pin_tokens = e.prefix->pin_at(cache_ids, npu::shared_prefix_len(lead, cache_ids)); } else { - pin_tokens = e.prefix->pin_point(e.prev_ids, ids); + pin_tokens = e.prefix->pin_point(e.prev_ids, cache_ids); } if (pin_tokens > 0 && int(pin_tokens) - 1 >= resume) pin_cut = int(pin_tokens) - 1; } + // One prefill call over prompt rows [off, off + take): token ids, or (pictures) those rows' embeddings and planes. + auto prefill = [&](int off, int take) -> bool { + if (!img) { + const std::vector part(ids.begin() + off, ids.begin() + off + take); + return e.fwd->step_batch(part, off, logits); + } + const size_t H = img_embeds.size() / ids.size(); + const std::vector rows(img_embeds.begin() + long(size_t(off) * H), img_embeds.begin() + long(size_t(off + take) * H)); + std::vector planes; + planes.reserve(size_t(3) * size_t(take)); + planes.insert(planes.end(), ex.positions.t.begin() + off, ex.positions.t.begin() + off + take); + planes.insert(planes.end(), ex.positions.h.begin() + off, ex.positions.h.begin() + off + take); + planes.insert(planes.end(), ex.positions.w.begin() + off, ex.positions.w.begin() + off + take); + Q4nxNpuForward::HybridInputs in; + in.position_ids = &planes; + return e.fwd->step_batch_embeds(rows, off, logits, in); + }; { const int chunk_max = std::max(1, e.fwd->max_batch_rows()); const int n = int(ids.size()); @@ -439,7 +528,7 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib Q4nxNpuForward::StateSnapshot snap; if (e.fwd->snapshot_state(off, snap)) { std::string why; - const std::vector shared(ids.begin(), ids.begin() + long(pin_tokens)); + const std::vector shared(cache_ids.begin(), cache_ids.begin() + long(pin_tokens)); const int slot = e.prefix->pin(shared, snap.bytes(), &why); if (slot >= 0 && size_t(slot) < e.prefix_state.size()) e.prefix_state[size_t(slot)] = std::move(snap); std::fprintf(stderr, "[prefix] pin slot=%d tokens=%zu state_at=%d %s\n", slot, pin_tokens, off, @@ -449,8 +538,7 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib } else if (pin_cut > off && off + take > pin_cut) { // Stop this chunk at the pin position; the next pass saves the state there. One more launch. take = pin_cut - off; - const std::vector head(ids.begin() + off, ids.begin() + off + take); - if (!e.fwd->step_batch(head, off, logits)) + if (!prefill(off, take)) throw std::runtime_error(std::string("forward step failed: ") + e.fwd->last_error()); off += take; continue; @@ -462,8 +550,7 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib // place, so a snapshot taken *after* the last token cannot be restored to just before it. // Splitting the final chunk costs one launch. if (off + take == n && take > 1) { - const std::vector head(ids.begin() + off, ids.begin() + off + take - 1); - if (!e.fwd->step_batch(head, off, logits)) + if (!prefill(off, take - 1)) throw std::runtime_error(std::string("forward step failed: ") + e.fwd->last_error()); off += take - 1; continue; @@ -472,7 +559,7 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib Q4nxNpuForward::StateSnapshot snap; if (e.fwd->snapshot_state(off, snap)) { std::string why; - const int slot = e.prefix->insert(ids, snap.bytes(), stats.prefix.slot, &why); + const int slot = e.prefix->insert(cache_ids, snap.bytes(), stats.prefix.slot, &why); if (slot >= 0 && size_t(slot) < e.prefix_state.size()) { e.prefix_state[size_t(slot)] = std::move(snap); std::fprintf(stderr, "[prefix] saved slot=%d tokens=%d bytes=%zu %s\n", slot, off, @@ -480,13 +567,12 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib } } } - const std::vector chunk(ids.begin() + off, ids.begin() + off + take); - if (!e.fwd->step_batch(chunk, off, logits)) + if (!prefill(off, take)) throw std::runtime_error(std::string("forward step failed: ") + e.fwd->last_error()); off += take; } } - if (e.prefix_pin) e.prev_ids = ids; + if (e.prefix_pin) e.prev_ids = cache_ids; // Prompt processing ends here: every prompt token has been through the model and the first // token's distribution is in `logits`. @@ -609,7 +695,7 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib // --spec-sample widens it to sampled requests, whose drafts are then judged by the distribution-keeping rule // instead (npu::SpecSampler): the reply follows the same distribution as the plain loop's, not its stream. const bool spec = (e.spec_lookup > 0 || e.draft) && (opts.sample.greedy() || e.spec_sample) && - e.dump_logits.empty(); + e.dump_logits.empty() && !img; // a picture's rotary positions are not the cache slots if (spec) { struct ServeBase : npu::mtp::BaseModel { Q4nxNpuForward& f; @@ -726,7 +812,8 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib // weight stream -- 8.7 s on the 27B -- spent on a result nobody reads. At // max_tokens=1 that is half the request's latency. if (n + 1 >= limit) break; - if (!e.fwd->step(next, (int)ids.size() + n, logits)) + if (!(img ? e.fwd->step(next, img_next + n, logits, (int)ids.size() + n) + : e.fwd->step(next, (int)ids.size() + n, logits))) throw std::runtime_error(std::string("forward step failed: ") + e.fwd->last_error()); } if (!stopped_at_stop) { @@ -1285,6 +1372,18 @@ int run_forward_serve(int argc, char** argv) { std::fprintf(stderr, "1bit forward-serve: dumping raw logits to %s\n", dump_logits.c_str()); } e.tok = std::make_unique(tok_json); + // engine#2324: a pack that ships a vision component (MiniCPM-V 4.7: vision_weight.q4nx beside model.q4nx) takes + // pictures on the chat route. A component that does not read is said once and leaves the route text-only. + if (std::filesystem::exists(std::filesystem::path(dir) / "vision_weight.q4nx")) { + try { + e.vision = std::make_unique(dir); + std::fprintf(stderr, "1bit forward-serve: vision component %s/vision_weight.q4nx: the chat route takes pictures " + "(data: URLs; the tower runs on the host)\n", dir.c_str()); + } catch (const std::exception& ex) { + e.vision.reset(); + std::fprintf(stderr, "1bit forward-serve: vision component not read (%s): pictures are refused\n", ex.what()); + } + } e.bos_id = json_int(cfg, "bos_token_id", -1); e.add_bos = json_int(cfg, "add_bos_token", 0) != 0; // config.json is the first source; what it does not state, tokenizer_config.json does (the BOS text, looked up diff --git a/app/npu_image_request.h b/app/npu_image_request.h new file mode 100644 index 000000000..047120242 --- /dev/null +++ b/app/npu_image_request.h @@ -0,0 +1,125 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// engine#2324: the pictures of an OpenAI chat request, as RGB bytes. +// +// A picture is a content part `{"type": "image_url", "image_url": {"url": "data:image/png;base64,..."}}` (or `"url"` as +// a bare string), in message order; the pack's chat template renders one `<|image_pad|>` for each, in the same order, +// which is how the k-th picture finds its place (npu/vision_request.h). Only `data:` URLs are taken: the route fetches +// nothing from the network. PNG, JPEG and binary PPM are decoded (stb_image, app/stb_image_impl.cpp); anything else, +// a picture larger than kMaxImagePixels, and a part with no URL are refused by name. +#pragma once + +#include "vision_request.h" + +#include + +#include +#include +#include +#include + +extern "C" { +unsigned char* stbi_load_from_memory(const unsigned char* buffer, int len, int* x, int* y, int* channels_in_file, + int desired_channels); +void stbi_image_free(void* retval_from_stbi_load); +const char* stbi_failure_reason(void); +} + +namespace onebit::npu { + +constexpr long kMaxImagePixels = 16L * 1024 * 1024; + +inline std::vector base64_decode(const std::string& s) { + static const auto value = [](char c) -> int { + if (c >= 'A' && c <= 'Z') return c - 'A'; + if (c >= 'a' && c <= 'z') return c - 'a' + 26; + if (c >= '0' && c <= '9') return c - '0' + 52; + if (c == '+' || c == '-') return 62; + if (c == '/' || c == '_') return 63; + return -1; + }; + std::vector out; + out.reserve(s.size() * 3 / 4); + uint32_t acc = 0; + int bits = 0; + for (char c : s) { + if (c == '=' || c == '\n' || c == '\r' || c == ' ') continue; + const int v = value(c); + if (v < 0) throw std::runtime_error("image_url: the data URL is not base64"); + acc = (acc << 6) | uint32_t(v); + bits += 6; + if (bits >= 8) { + bits -= 8; + out.push_back(uint8_t((acc >> bits) & 0xFF)); + } + } + return out; +} + +// RGB bytes from encoded PNG / JPEG / PPM bytes. +inline vision::RgbImage decode_image_bytes(const std::vector& bytes) { + if (bytes.empty() || bytes.size() > size_t(1) << 30) throw std::runtime_error("image_url: an empty or oversized image"); + int w = 0, h = 0, ch = 0; + unsigned char* px = stbi_load_from_memory(bytes.data(), int(bytes.size()), &w, &h, &ch, 3); + if (!px) throw std::runtime_error(std::string("image_url: not a PNG, JPEG or PPM image (") + stbi_failure_reason() + ")"); + vision::RgbImage im; + im.height = h; + im.width = w; + const bool too_big = long(w) * long(h) > kMaxImagePixels; + if (!too_big) im.rgb.assign(px, px + size_t(w) * size_t(h) * 3); + stbi_image_free(px); + if (too_big) + throw std::runtime_error("image_url: a " + std::to_string(w) + " x " + std::to_string(h) + " image is more than " + + std::to_string(kMaxImagePixels) + " pixels"); + return im; +} + +inline vision::RgbImage image_from_data_url(const std::string& url) { + if (url.rfind("data:", 0) != 0) + throw std::runtime_error("image_url: only data: URLs are taken (this route fetches nothing from the network)"); + const size_t comma = url.find(','); + if (comma == std::string::npos || url.substr(0, comma).find(";base64") == std::string::npos) + throw std::runtime_error("image_url: a data URL must be base64 (data:image/png;base64,...)"); + return decode_image_bytes(base64_decode(url.substr(comma + 1))); +} + +// Every picture in `messages`, in order. True when there is at least one. +template +inline bool request_images(const Json& messages, std::vector& out) { + out.clear(); + if (!messages.is_array()) return false; + for (const auto& m : messages) { + if (!m.is_object() || !m.contains("content") || !m["content"].is_array()) continue; + for (const auto& part : m["content"]) { + if (!part.is_object()) continue; + const std::string type = part.contains("type") && part["type"].is_string() ? part["type"].template get() : ""; + // the set the MiniCPM-V template turns into a pad: an `image` or `image_url` key, or type "image" + if (type != "image_url" && type != "image" && !part.contains("image_url") && !part.contains("image")) continue; + std::string url; + if (part.contains("image_url")) { + const auto& iu = part["image_url"]; + if (iu.is_string()) url = iu.template get(); + else if (iu.is_object() && iu.contains("url") && iu["url"].is_string()) url = iu["url"].template get(); + } + if (url.empty() && part.contains("image") && part["image"].is_string()) url = part["image"].template get(); + if (url.empty()) throw std::runtime_error("image_url: a picture part without a url"); + out.push_back(image_from_data_url(url)); + } + } + return !out.empty(); +} + +} // namespace onebit::npu diff --git a/app/stb_image_impl.cpp b/app/stb_image_impl.cpp new file mode 100644 index 000000000..865823051 --- /dev/null +++ b/app/stb_image_impl.cpp @@ -0,0 +1,26 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// engine#2324: the one translation unit that compiles stb_image. Only the formats a chat client sends (PNG, JPEG) and +// PNM (the probes' binary PPM) are built; the rest of stb_image's decoders are left out, and so is its file I/O -- +// the route decodes bytes it already holds. +#define STB_IMAGE_IMPLEMENTATION +#define STBI_ONLY_PNG +#define STBI_ONLY_JPEG +#define STBI_ONLY_PNM +#define STBI_NO_STDIO +#define STBI_NO_LINEAR +#define STBI_NO_HDR +#include "stb_image.h" diff --git a/docs/npu.md b/docs/npu.md index 9eedb6c1d..64c1b8dc0 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -2039,12 +2039,57 @@ decoder is involved, and the prompt is the one the pack's chat template renders with one image and thinking off, written out in the probe rather than rendered. The text model opens the device unless `NPU_INFER_HOST_GEMM` is set. -It is a probe, not a route: `forward-serve` does not accept an image, and the tower runs on the -CPU (about 3 s per 448 × 448 view on 16 threads, 18 s on one). The measured rows are in +The probe is the bench for the route below. Its tower runs on the CPU (about 3 s per 448 × 448 view on +16 threads, 18 s on one). The measured rows are in engine#1788. On the published packs the text model only loads once the padded-head rule of engine#2026 is in: their head is stored 2 and 16 rows past the vocabulary, and without that rule the probe stops at `text model: lm_head dequant failed`. +**Pictures on the chat route (engine#2324).** A pack with a vision component (`vision_weight.q4nx` beside +`model.q4nx`) takes pictures through `1bit serve -m --device npu`, as OpenAI content parts: + +``` +{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "data:image/png;base64,..."}}, + {"type": "text", "text": "What colour is this image?"}]} +``` + +- Only `data:` URLs are taken; the route fetches nothing from the network. PNG, JPEG and binary PPM are decoded + (`stb_image`, pinned in `CMakeLists.txt`, built with those three formats only). A picture over 16 M pixels, a + part with no URL, and a pack with no vision component are each a 400 naming the reason. +- The pack's own template renders one `<|image_pad|>` per picture, in order. `npu/vision_request.h` replaces the + k-th pad with `k` and the picture's tokens, runs the tower on each piece (on the host, before + the engine lock), and lays out the three position planes. The prompt then goes through `step_batch_embeds` in + the route's usual chunks, with the tower's rows where the placeholders are. Decoding resumes at the planes' next + position, while the cache slot counts from the prompt's length. +- The prompt cache is keyed on the ids with every image row replaced by a key derived from that picture's bytes. + Two pictures of one size expand to identical ids, and an ids-only key would hand one picture's state to the + other. Measured on the 1B (host GEMM): red, then blue at the same size, then red again answered `Red`, `Blue`, + `Red`. With a 672 × 1344 picture (486 prompt rows, 444 of them the picture's), it hits as it should: + + | request | answer | time | cached | + |---|---|---|---| + | red | `red` | 381.3 s | 0 | + | the same picture again | `red` | 48.2 s | 485 of 486 | + | blue, same size | `Blue` | 382.9 s | 0 | + + That host run still ran the tower on a hit. The route now expands a picture's ids, positions and key from its plan + alone, looks the prompt up first, and runs the tower only when a picture row lies past the resume point. On the + NPU (`npu-hold`, amdxdna clean before and after), the same three requests: + + | request | answer | time | cached | + |---|---|---|---| + | red | `Red` | 43.0 s | 0 | + | the same picture again | `Red` | 1.1 s | 485 of 486 (the tower is not run) | + | blue, same size | `Blue` | 55.5 s | 0 | + + Prompts under the cache's `--prefix-min-gain` (128 tokens) are not saved at all, picture or not: a 448 × 448 + picture with a short question is 93 rows. +- Speculative decoding is off for a request with pictures: its rotary positions are not the cache slots the + drafter counts. + +`tests/npu_image_request_test.cpp` decodes crafted PPM and PNG data URLs to exactly their pixels, keeps request +order and names each refusal. + Three options exist to bisect a wrong answer against a reference run of the same prompt, and are how the missing q/k norm above was found: diff --git a/npu/vision_request.cpp b/npu/vision_request.cpp new file mode 100644 index 000000000..3e5cf9406 --- /dev/null +++ b/npu/vision_request.cpp @@ -0,0 +1,134 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "vision_request.h" + +#include "vision_image.h" +#include "vision_plan.h" +#include "vision_tower.h" + +#include + +namespace onebit::npu::vision { + +bool image_markers(const Tokenizer& tok, ImageMarkers& m, int& id_open, int& id_close, std::string& why) { + m.image_start = tok.token_id(""); + m.image_end = tok.token_id(""); + m.slice_start = tok.token_id(""); + m.slice_end = tok.token_id(""); + m.image_pad = tok.token_id("<|image_pad|>"); + id_open = tok.token_id(""); + id_close = tok.token_id(""); + const std::vector nl = tok.encode("\n"); + if (m.image_start < 0 || m.image_end < 0 || m.slice_start < 0 || m.slice_end < 0 || m.image_pad < 0 || id_open < 0 || + id_close < 0 || nl.size() != 1) { + why = "this tokenizer lacks one of , , , , <|image_pad|>, , , or " + "does not encode a newline as one token"; + return false; + } + m.newline = nl[0]; + return true; +} + +uint64_t images_hash(const std::vector& images) { + uint64_t h = 1469598103934665603ull; + auto mix = [&](const uint8_t* p, size_t n) { + for (size_t i = 0; i < n; i++) { h ^= p[i]; h *= 1099511628211ull; } + }; + for (const RgbImage& im : images) { + const int32_t dims[2] = {im.height, im.width}; + mix(reinterpret_cast(dims), sizeof dims); + mix(im.rgb.data(), im.rgb.size()); + } + return h; +} + +bool tower_rows(const Pack& pack, const std::vector& images, ExpandedPrompt& out, std::string& why) { + out.rows.clear(); + try { + for (const RgbImage& im : images) { + const ImagePlan plan = plan_image(im.height, im.width); + for (const Piece& p : preprocess(im.rgb.data(), im.height, im.width, plan)) { + const std::vector r = tower_forward(pack, p.pixels.data(), p.grid_h, p.grid_w); + out.rows.insert(out.rows.end(), r.begin(), r.end()); + } + } + } catch (const std::exception& e) { + why = e.what(); + return false; + } + size_t placeholders = 0; + for (int t : out.ids) placeholders += t == out.image_pad; + if (out.width <= 0 || placeholders * size_t(out.width) != out.rows.size()) { + why = "the tower produced " + std::to_string(out.rows.size()) + " values for " + std::to_string(placeholders) + + " placeholder rows of width " + std::to_string(out.width); + return false; + } + return true; +} + +bool expand_images(const Tokenizer& tok, const Pack& pack, const std::vector& rendered, const std::vector& images, + ExpandedPrompt& out, std::string& why, int max_rows, bool run_tower) { + out = ExpandedPrompt{}; + ImageMarkers m; + int id_open = -1, id_close = -1; + if (!image_markers(tok, m, id_open, id_close, why)) return false; + size_t pads = 0; + for (int t : rendered) pads += t == m.image_pad; + if (pads != images.size()) { + why = "the rendered prompt holds " + std::to_string(pads) + " image placeholder(s) for " + std::to_string(images.size()) + + " image(s)"; + return false; + } + out.width = pack.geometry().out_hidden; + out.image_pad = m.image_pad; + size_t k = 0; + try { + for (int t : rendered) { + if (t != m.image_pad) { out.ids.push_back(t); continue; } + const RgbImage& im = images[k]; + if (im.height <= 0 || im.width <= 0 || im.rgb.size() != size_t(im.height) * size_t(im.width) * 3) { + why = "image " + std::to_string(k) + " is not height x width x 3 bytes"; + return false; + } + const ImagePlan plan = plan_image(im.height, im.width); + if (plan.total_rows() > max_rows) { + why = "image " + std::to_string(k) + " (" + std::to_string(im.height) + " x " + std::to_string(im.width) + ") is " + + std::to_string(plan.total_rows()) + " rows; this route takes at most " + std::to_string(max_rows); + return false; + } + // Every piece's grid comes from the plan: the whole view, then each slice (all the same size). + out.grids.push_back({plan.source.grid_h, plan.source.grid_w}); + for (int sl = 0; sl < plan.slice_rows * plan.slice_cols; ++sl) out.grids.push_back({plan.slice.grid_h, plan.slice.grid_w}); + // `k` is ordinary text in front of the image, as the reference processor writes it. + out.ids.push_back(id_open); + const std::vector num = tok.encode(std::to_string(k)); + out.ids.insert(out.ids.end(), num.begin(), num.end()); + out.ids.push_back(id_close); + const std::vector img = image_tokens(plan, m); + out.ids.insert(out.ids.end(), img.begin(), img.end()); + ++k; + } + out.positions = prompt_positions(out.ids, m, out.grids); + } catch (const std::exception& e) { + why = e.what(); + return false; + } + out.image_hash = images_hash(images); + out.images = int(images.size()); + return run_tower ? tower_rows(pack, images, out, why) : true; +} + +} // namespace onebit::npu::vision diff --git a/npu/vision_request.h b/npu/vision_request.h new file mode 100644 index 000000000..a033a988c --- /dev/null +++ b/npu/vision_request.h @@ -0,0 +1,74 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// engine#2324: a chat request's pictures, from decoded RGB to what the text model is given -- the steps +// minicpm_image_probe chains, as one call a serve route can make. +// +// The pack's chat template renders one `<|image_pad|>` per image content part, in order. expand_images() replaces the +// k-th pad with `k` and the image's own tokens (vision::image_tokens: ``, one placeholder per +// row of the whole view, ``, then the slices), runs the tower on every piece (the whole view, then the slices), +// and lays out the three position planes (vision::prompt_positions). The caller puts the tower's rows where the +// placeholders are and its own embedding everywhere else, and sends the prompt through step_batch_embeds with the +// planes. Decoding then resumes at `positions.next` on the rotary planes while the cache slot keeps counting from the +// prompt's length: an image moves the position by its side, not by its row count. +// +// `image_hash` covers every image's size and bytes in order. Two pictures of one size expand to identical ids, so a +// prompt cache keyed on ids alone would hand one picture's state to the other; the hash is what tells them apart. +#pragma once + +#include "tokenizer.h" +#include "vision_pack.h" +#include "vision_prompt.h" + +#include +#include +#include +#include + +namespace onebit::npu::vision { + +struct RgbImage { + int height = 0, width = 0; + std::vector rgb; // height * width * 3, R G B +}; + +struct ExpandedPrompt { + std::vector ids; // the rendered prompt with every pad expanded + std::vector rows; // the tower's rows, every piece in order, `width` floats each + int width = 0; // the text model's hidden size, as the tower projects + PromptPositions positions; // (t, h, w) per id, and the rotary position decoding resumes at + std::vector> grids; // each piece's patch grid, as prompt_positions read them + int image_pad = -1; // the placeholder id: a row of `rows` goes wherever it appears + uint64_t image_hash = 0; // FNV-1a over every image's height, width and bytes, in order + int images = 0; +}; + +// The markers a MiniCPM-V 4.7 tokenizer carries; false (with `why`) when one is missing. +bool image_markers(const Tokenizer& tok, ImageMarkers& m, int& id_open, int& id_close, std::string& why); + +// Expand `rendered` (the template's ids, one pad per image) with `images`, in order. False with `why` when the pad +// count and the image count differ, a marker is missing, or an image would exceed `max_rows` placeholder rows. +// With `run_tower` false the ids, grids, positions and hash are filled and `rows` is left empty: everything a prompt +// cache needs to decide a hit, without the tower (the plan alone gives every piece's grid). tower_rows() fills them. +bool expand_images(const Tokenizer& tok, const Pack& pack, const std::vector& rendered, const std::vector& images, + ExpandedPrompt& out, std::string& why, int max_rows = 4096, bool run_tower = true); + +// The tower's rows for `images` into `out.rows` (an ExpandedPrompt made with run_tower false), checked against its +// placeholder count. +bool tower_rows(const Pack& pack, const std::vector& images, ExpandedPrompt& out, std::string& why); + +uint64_t images_hash(const std::vector& images); + +} // namespace onebit::npu::vision diff --git a/tests/npu_image_request_test.cpp b/tests/npu_image_request_test.cpp new file mode 100644 index 000000000..38ef67b10 --- /dev/null +++ b/tests/npu_image_request_test.cpp @@ -0,0 +1,118 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// engine#2324: a chat request's pictures to RGB bytes (app/npu_image_request.h). A crafted PPM and a crafted PNG (stored +// zlib blocks, no compression needed) go in as data URLs; the bytes that come out must be exactly the pixels written. +#include "npu_image_request.h" + +#include +#include +#include + +static int failures = 0; +static void check(bool ok, const char* what) { + if (!ok) { std::printf("FAIL %s\n", what); failures++; } +} + +static std::string b64(const std::vector& d) { + static const char* A = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + std::string o; + for (size_t i = 0; i < d.size(); i += 3) { + uint32_t v = uint32_t(d[i]) << 16 | (i + 1 < d.size() ? uint32_t(d[i + 1]) << 8 : 0) | (i + 2 < d.size() ? d[i + 2] : 0); + o += A[(v >> 18) & 63]; o += A[(v >> 12) & 63]; + o += i + 1 < d.size() ? A[(v >> 6) & 63] : '='; o += i + 2 < d.size() ? A[v & 63] : '='; + } + return o; +} + +static uint32_t crc32(const uint8_t* p, size_t n, uint32_t c = 0xFFFFFFFFu) { + for (size_t i = 0; i < n; i++) { c ^= p[i]; for (int k = 0; k < 8; k++) c = (c >> 1) ^ (0xEDB88320u & (0u - (c & 1))); } + return c; +} +static uint32_t adler32(const std::vector& d) { + uint32_t a = 1, b = 0; + for (uint8_t x : d) { a = (a + x) % 65521; b = (b + a) % 65521; } + return b << 16 | a; +} +static void be32(std::vector& o, uint32_t v) { for (int s = 24; s >= 0; s -= 8) o.push_back(uint8_t(v >> s)); } +static void chunk(std::vector& o, const char* type, const std::vector& data) { + be32(o, uint32_t(data.size())); + std::vector td(type, type + 4); + td.insert(td.end(), data.begin(), data.end()); + o.insert(o.end(), td.begin(), td.end()); + be32(o, crc32(td.data(), td.size()) ^ 0xFFFFFFFFu); +} +// An RGB PNG with filter 0 rows in one stored (uncompressed) deflate block. +static std::vector png(int w, int h, const std::vector& rgb) { + std::vector raw; + for (int y = 0; y < h; y++) { raw.push_back(0); raw.insert(raw.end(), rgb.begin() + y * w * 3, rgb.begin() + (y + 1) * w * 3); } + std::vector z = {0x78, 0x01, 0x01}; + const uint16_t len = uint16_t(raw.size()); + z.push_back(uint8_t(len)); z.push_back(uint8_t(len >> 8)); z.push_back(uint8_t(~len)); z.push_back(uint8_t(~len >> 8)); + z.insert(z.end(), raw.begin(), raw.end()); + be32(z, adler32(raw)); + std::vector o = {0x89, 'P', 'N', 'G', 0x0D, 0x0A, 0x1A, 0x0A}; + std::vector ihdr; + be32(ihdr, uint32_t(w)); be32(ihdr, uint32_t(h)); + ihdr.insert(ihdr.end(), {8, 2, 0, 0, 0}); + chunk(o, "IHDR", ihdr); chunk(o, "IDAT", z); chunk(o, "IEND", {}); + return o; +} + +int main() { + using namespace onebit::npu; + const int w = 5, h = 3; + std::vector rgb(size_t(w) * h * 3); + for (size_t i = 0; i < rgb.size(); i++) rgb[i] = uint8_t(i * 37 + 11); + + // PPM (P6): what the probes read + std::string hdr = "P6\n" + std::to_string(w) + " " + std::to_string(h) + "\n255\n"; + std::vector ppm(hdr.begin(), hdr.end()); + ppm.insert(ppm.end(), rgb.begin(), rgb.end()); + vision::RgbImage a = image_from_data_url("data:image/x-portable-pixmap;base64," + b64(ppm)); + check(a.height == h && a.width == w && a.rgb == rgb, "a PPM data URL decodes to exactly its pixels"); + + // PNG: what a chat client sends + vision::RgbImage b = image_from_data_url("data:image/png;base64," + b64(png(w, h, rgb))); + check(b.height == h && b.width == w && b.rgb == rgb, "a PNG data URL decodes to exactly its pixels"); + + // the request shape: content parts in message order, both url spellings, text parts skipped + nlohmann::json msgs = nlohmann::json::parse(R"([ + {"role": "system", "content": "be brief"}, + {"role": "user", "content": [{"type": "text", "text": "first"}, {"type": "image_url", "image_url": {"url": "X"}}]}, + {"role": "user", "content": [{"type": "image_url", "image_url": "Y"}]}])"); + msgs[1]["content"][1]["image_url"]["url"] = "data:image/png;base64," + b64(png(w, h, rgb)); + msgs[2]["content"][0]["image_url"] = "data:image/x-portable-pixmap;base64," + b64(ppm); + std::vector got; + check(request_images(msgs, got) && got.size() == 2 && got[0].rgb == rgb && got[1].rgb == rgb, + "request_images finds both pictures, in order, in either url spelling"); + check(!request_images(nlohmann::json::parse(R"([{"role":"user","content":"no pictures"}])"), got) && got.empty(), + "a text-only request has no pictures"); + + // refusals, by name + auto refuses = [](const std::string& url, const char* needle) { + try { image_from_data_url(url); } catch (const std::exception& e) { return std::string(e.what()).find(needle) != std::string::npos; } + return false; + }; + check(refuses("https://example.com/cat.png", "only data: URLs"), "a network URL is refused"); + check(refuses("data:image/png,rawbytes", "must be base64"), "a non-base64 data URL is refused"); + check(refuses("data:image/png;base64," + b64({1, 2, 3, 4}), "not a PNG, JPEG or PPM"), "bytes that are no image are refused"); + check(vision::images_hash({a}) == vision::images_hash({b}) && vision::images_hash({a}) != vision::images_hash({vision::RgbImage{h, w, std::vector(rgb.size(), 0)}}), + "the image hash follows the pixels, not the encoding"); + + if (failures) { std::printf("%d failure(s)\n", failures); return 1; } + std::printf("ok: PPM and PNG data URLs decode exactly, request order kept, refusals named\n"); + return 0; +} From 8b31e505f7f0568b043bbd517ad3be727829f7b2 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Fri, 13 Nov 2026 01:10:50 -0400 Subject: [PATCH 2/3] NOTICE: stb_image, fetched and compiled in for pictures; the picture key's width noted (refs #2324) Review of #2335: the new dependency had no NOTICE block. nothings/stb at the pinned commit, PNG/JPEG/PNM only, MIT or public domain. The cache key's comment now says it is 30 bits of the 64-bit hash, mixed per row. Co-Authored-By: Claude Opus 5.5 --- NOTICE | 5 +++++ app/forward_serve.cpp | 2 ++ 2 files changed, 7 insertions(+) diff --git a/NOTICE b/NOTICE index c1628a9a7..b51a425be 100644 --- a/NOTICE +++ b/NOTICE @@ -167,6 +167,11 @@ cpp-httplib (v0.47.0) Copyright (c) 2017 yhirose License: MIT +stb_image (nothings/stb 9600522; PNG, JPEG and PNM only, app/stb_image_impl.cpp) + https://github.com/nothings/stb + Copyright (c) 2017 Sean Barrett + License: MIT (or public domain, at the user's choice) + ------------------------------------------------------------------------------- Linked at build time against a system package (not redistributed) ------------------------------------------------------------------------------- diff --git a/app/forward_serve.cpp b/app/forward_serve.cpp index 75defd681..56a44667d 100644 --- a/app/forward_serve.cpp +++ b/app/forward_serve.cpp @@ -399,6 +399,8 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib if (ids[i] == ex.image_pad) { // A key id no token has (negative), and a different one for a different picture: two pictures of one // size expand to identical ids, and an ids-only key would hand one picture's state to the other. + // The key is 30 bits of the 64-bit FNV-1a hash, mixed with the row: a false hit between two pictures + // needs those 30 bits to agree on every one of their rows, which is a cache-key risk, not a token one. key_ids[i] = -int(1 + ((ex.image_hash ^ (uint64_t(row) * 0x9E3779B97F4A7C15ull)) & 0x3FFFFFFF)); ++row; } From b2662bf40ee05a54383e6213f80f758b58779338 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Fri, 13 Nov 2026 01:11:06 -0400 Subject: [PATCH 3/3] forward-serve: the picture key's collision note says what it is: one 30-bit collision (refs #2324) Co-Authored-By: Claude Opus 5.5 --- app/forward_serve.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/forward_serve.cpp b/app/forward_serve.cpp index 56a44667d..793336d00 100644 --- a/app/forward_serve.cpp +++ b/app/forward_serve.cpp @@ -399,8 +399,9 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib if (ids[i] == ex.image_pad) { // A key id no token has (negative), and a different one for a different picture: two pictures of one // size expand to identical ids, and an ids-only key would hand one picture's state to the other. - // The key is 30 bits of the 64-bit FNV-1a hash, mixed with the row: a false hit between two pictures - // needs those 30 bits to agree on every one of their rows, which is a cache-key risk, not a token one. + // The key keeps 30 bits of the 64-bit FNV-1a hash. The row is mixed in the same way for every picture, + // so two pictures share keys exactly when their hashes agree in those 30 bits: a false hit is one + // 30-bit collision (about 1 in 1e9 per pair), acceptable for a cache key. key_ids[i] = -int(1 + ((ex.image_hash ^ (uint64_t(row) * 0x9E3779B97F4A7C15ull)) & 0x3FFFFFFF)); ++row; }