Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 15 additions & 3 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,12 @@ set(HTTPLIB_INSTALL OFF CACHE BOOL "" FORCE)
FetchContent_Declare(httplib
GIT_REPOSITORY https://github.com/yhirose/cpp-httplib.git
GIT_TAG fe332fa06bac76a1c6d402c08f414052999347da) # v0.47.0
FetchContent_MakeAvailable(json httplib)
# engine#2324: images in a chat request arrive as PNG or JPEG bytes; stb_image decodes them (one header, public domain /
# MIT). It has no CMakeLists, so it is only fetched; app/stb_image_impl.cpp compiles it with PNG, JPEG and PNM only.
FetchContent_Declare(stb
GIT_REPOSITORY https://github.com/nothings/stb.git
GIT_TAG 9600522de53973336fa033a289858a4354cbb04c)
FetchContent_MakeAvailable(json httplib stb)

# config/laya-first.json (the routing table, docs/laya-first.md §4) is compiled in like
# config/recipes.json; `1bit laya-first --laya-first FILE` replaces it at run time.
Expand Down Expand Up @@ -903,7 +908,7 @@ option(ONEBIT_NPU_HOST "Build the NPU host library and its tests" ${_onebit_npu_
if(ONEBIT_NPU_HOST)
find_package(PkgConfig REQUIRED)
pkg_check_modules(PCRE2 REQUIRED IMPORTED_TARGET libpcre2-8)
add_library(onebit_npu STATIC npu/file_map.cpp npu/flm_head_import.cpp npu/full_elf.cpp npu/model.cpp npu/vision_pack.cpp npu/vision_tower.cpp npu/vision_image.cpp
add_library(onebit_npu STATIC npu/file_map.cpp npu/flm_head_import.cpp npu/full_elf.cpp npu/model.cpp npu/vision_pack.cpp npu/vision_tower.cpp npu/vision_image.cpp npu/vision_request.cpp
npu/audio_exec.cpp npu/moe.cpp npu/pack.cpp npu/pack_import.cpp npu/private_route.cpp
npu/q4nx.cpp npu/q4nx_device.cpp npu/tokenizer.cpp)
target_include_directories(onebit_npu PUBLIC npu)
Expand Down Expand Up @@ -1452,6 +1457,12 @@ add_executable(npu_vision_image_test tests/npu_vision_image_test.cpp)
target_link_libraries(npu_vision_image_test PRIVATE onebit_npu)
add_test(NAME npu_vision_image COMMAND npu_vision_image_test)
add_dependencies(onebit npu_vision_image_test)
# engine#2324: a chat request's pictures (data URLs, PNG / JPEG / PPM) to RGB bytes, exactly.
add_executable(npu_image_request_test tests/npu_image_request_test.cpp app/stb_image_impl.cpp)
target_include_directories(npu_image_request_test PRIVATE app npu ${stb_SOURCE_DIR})
target_link_libraries(npu_image_request_test PRIVATE onebit_npu nlohmann_json::nlohmann_json)
add_test(NAME npu_image_request COMMAND npu_image_request_test)
add_dependencies(onebit npu_image_request_test)
# The text side of an image: its tokens and the canvas positions, against the reference model code's own answers.
add_executable(npu_vision_prompt_test tests/npu_vision_prompt_test.cpp)
target_include_directories(npu_vision_prompt_test PRIVATE npu)
Expand Down Expand Up @@ -2105,7 +2116,8 @@ endif()
# The library body is onebit_npu_forward_library() (defined above the runtime options), so the CI build of the
# same sources against the runtime stub (engine#1946) cannot drift from this one.
onebit_npu_forward_library(onebit_npu_forward onebit_npu_rt)
target_sources(onebit PRIVATE app/forward_serve.cpp)
target_sources(onebit PRIVATE app/forward_serve.cpp app/stb_image_impl.cpp) # engine#2324: stb_image for pictures
target_include_directories(onebit PRIVATE ${stb_SOURCE_DIR})
# engine#1781: forward-serve --spec-draft runs a DFlash draft package (npu/dflash_head.h) as its drafter.
target_sources(onebit PRIVATE npu/dflash_forward.cpp npu/dflash_draft.cpp)
# engine#2111: the usage object's pure function, its own TU so tests/serve_usage_test.cpp can link it without
Expand Down
5 changes: 5 additions & 0 deletions NOTICE
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,11 @@ cpp-httplib (v0.47.0)
Copyright (c) 2017 yhirose
License: MIT

stb_image (nothings/stb 9600522; PNG, JPEG and PNM only, app/stb_image_impl.cpp)
https://github.com/nothings/stb
Copyright (c) 2017 Sean Barrett
License: MIT (or public domain, at the user's choice)

-------------------------------------------------------------------------------
Linked at build time against a system package (not redistributed)
-------------------------------------------------------------------------------
Expand Down
132 changes: 117 additions & 15 deletions app/forward_serve.cpp

Large diffs are not rendered by default.

125 changes: 125 additions & 0 deletions app/npu_image_request.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
// Copyright 2026 bong-water-water-bong
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// engine#2324: the pictures of an OpenAI chat request, as RGB bytes.
//
// A picture is a content part `{"type": "image_url", "image_url": {"url": "data:image/png;base64,..."}}` (or `"url"` as
// a bare string), in message order; the pack's chat template renders one `<|image_pad|>` for each, in the same order,
// which is how the k-th picture finds its place (npu/vision_request.h). Only `data:` URLs are taken: the route fetches
// nothing from the network. PNG, JPEG and binary PPM are decoded (stb_image, app/stb_image_impl.cpp); anything else,
// a picture larger than kMaxImagePixels, and a part with no URL are refused by name.
#pragma once

#include "vision_request.h"

#include <nlohmann/json.hpp>

#include <cstdint>
#include <stdexcept>
#include <string>
#include <vector>

extern "C" {
unsigned char* stbi_load_from_memory(const unsigned char* buffer, int len, int* x, int* y, int* channels_in_file,
int desired_channels);
void stbi_image_free(void* retval_from_stbi_load);
const char* stbi_failure_reason(void);
}

namespace onebit::npu {

constexpr long kMaxImagePixels = 16L * 1024 * 1024;

inline std::vector<uint8_t> base64_decode(const std::string& s) {
static const auto value = [](char c) -> int {
if (c >= 'A' && c <= 'Z') return c - 'A';
if (c >= 'a' && c <= 'z') return c - 'a' + 26;
if (c >= '0' && c <= '9') return c - '0' + 52;
if (c == '+' || c == '-') return 62;
if (c == '/' || c == '_') return 63;
return -1;
};
std::vector<uint8_t> out;
out.reserve(s.size() * 3 / 4);
uint32_t acc = 0;
int bits = 0;
for (char c : s) {
if (c == '=' || c == '\n' || c == '\r' || c == ' ') continue;
const int v = value(c);
if (v < 0) throw std::runtime_error("image_url: the data URL is not base64");
acc = (acc << 6) | uint32_t(v);
bits += 6;
if (bits >= 8) {
bits -= 8;
out.push_back(uint8_t((acc >> bits) & 0xFF));
}
}
return out;
}

// RGB bytes from encoded PNG / JPEG / PPM bytes.
inline vision::RgbImage decode_image_bytes(const std::vector<uint8_t>& bytes) {
if (bytes.empty() || bytes.size() > size_t(1) << 30) throw std::runtime_error("image_url: an empty or oversized image");
int w = 0, h = 0, ch = 0;
unsigned char* px = stbi_load_from_memory(bytes.data(), int(bytes.size()), &w, &h, &ch, 3);
if (!px) throw std::runtime_error(std::string("image_url: not a PNG, JPEG or PPM image (") + stbi_failure_reason() + ")");
vision::RgbImage im;
im.height = h;
im.width = w;
const bool too_big = long(w) * long(h) > kMaxImagePixels;
if (!too_big) im.rgb.assign(px, px + size_t(w) * size_t(h) * 3);
stbi_image_free(px);
if (too_big)
throw std::runtime_error("image_url: a " + std::to_string(w) + " x " + std::to_string(h) + " image is more than " +
std::to_string(kMaxImagePixels) + " pixels");
return im;
}

inline vision::RgbImage image_from_data_url(const std::string& url) {
if (url.rfind("data:", 0) != 0)
throw std::runtime_error("image_url: only data: URLs are taken (this route fetches nothing from the network)");
const size_t comma = url.find(',');
if (comma == std::string::npos || url.substr(0, comma).find(";base64") == std::string::npos)
throw std::runtime_error("image_url: a data URL must be base64 (data:image/png;base64,...)");
return decode_image_bytes(base64_decode(url.substr(comma + 1)));
}

// Every picture in `messages`, in order. True when there is at least one.
template <class Json>
inline bool request_images(const Json& messages, std::vector<vision::RgbImage>& out) {
out.clear();
if (!messages.is_array()) return false;
for (const auto& m : messages) {
if (!m.is_object() || !m.contains("content") || !m["content"].is_array()) continue;
for (const auto& part : m["content"]) {
if (!part.is_object()) continue;
const std::string type = part.contains("type") && part["type"].is_string() ? part["type"].template get<std::string>() : "";
// the set the MiniCPM-V template turns into a pad: an `image` or `image_url` key, or type "image"
if (type != "image_url" && type != "image" && !part.contains("image_url") && !part.contains("image")) continue;
std::string url;
if (part.contains("image_url")) {
const auto& iu = part["image_url"];
if (iu.is_string()) url = iu.template get<std::string>();
else if (iu.is_object() && iu.contains("url") && iu["url"].is_string()) url = iu["url"].template get<std::string>();
}
if (url.empty() && part.contains("image") && part["image"].is_string()) url = part["image"].template get<std::string>();
if (url.empty()) throw std::runtime_error("image_url: a picture part without a url");
out.push_back(image_from_data_url(url));
}
}
return !out.empty();
}

} // namespace onebit::npu
26 changes: 26 additions & 0 deletions app/stb_image_impl.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
// Copyright 2026 bong-water-water-bong
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// engine#2324: the one translation unit that compiles stb_image. Only the formats a chat client sends (PNG, JPEG) and
// PNM (the probes' binary PPM) are built; the rest of stb_image's decoders are left out, and so is its file I/O --
// the route decodes bytes it already holds.
#define STB_IMAGE_IMPLEMENTATION
#define STBI_ONLY_PNG
#define STBI_ONLY_JPEG
#define STBI_ONLY_PNM
#define STBI_NO_STDIO
#define STBI_NO_LINEAR
#define STBI_NO_HDR
#include "stb_image.h"
49 changes: 47 additions & 2 deletions docs/npu.md
Original file line number Diff line number Diff line change
Expand Up @@ -2039,12 +2039,57 @@ decoder is involved, and the prompt is the one the pack's chat template renders
with one image and thinking off, written out in the probe rather than rendered. The text model opens
the device unless `NPU_INFER_HOST_GEMM` is set.

It is a probe, not a route: `forward-serve` does not accept an image, and the tower runs on the
CPU (about 3 s per 448 × 448 view on 16 threads, 18 s on one). The measured rows are in
The probe is the bench for the route below. Its tower runs on the CPU (about 3 s per 448 × 448 view on
16 threads, 18 s on one). The measured rows are in
engine#1788. On the published packs the text model only loads once the padded-head rule of
engine#2026 is in: their head is stored 2 and 16 rows past the vocabulary, and without that rule the
probe stops at `text model: lm_head dequant failed`.

**Pictures on the chat route (engine#2324).** A pack with a vision component (`vision_weight.q4nx` beside
`model.q4nx`) takes pictures through `1bit serve -m <pack dir> --device npu`, as OpenAI content parts:

```
{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "data:image/png;base64,..."}},
{"type": "text", "text": "What colour is this image?"}]}
```

- Only `data:` URLs are taken; the route fetches nothing from the network. PNG, JPEG and binary PPM are decoded
(`stb_image`, pinned in `CMakeLists.txt`, built with those three formats only). A picture over 16 M pixels, a
part with no URL, and a pack with no vision component are each a 400 naming the reason.
- The pack's own template renders one `<|image_pad|>` per picture, in order. `npu/vision_request.h` replaces the
k-th pad with `<image_id>k</image_id>` and the picture's tokens, runs the tower on each piece (on the host, before
the engine lock), and lays out the three position planes. The prompt then goes through `step_batch_embeds` in
the route's usual chunks, with the tower's rows where the placeholders are. Decoding resumes at the planes' next
position, while the cache slot counts from the prompt's length.
- The prompt cache is keyed on the ids with every image row replaced by a key derived from that picture's bytes.
Two pictures of one size expand to identical ids, and an ids-only key would hand one picture's state to the
other. Measured on the 1B (host GEMM): red, then blue at the same size, then red again answered `Red`, `Blue`,
`Red`. With a 672 × 1344 picture (486 prompt rows, 444 of them the picture's), it hits as it should:

| request | answer | time | cached |
|---|---|---|---|
| red | `red` | 381.3 s | 0 |
| the same picture again | `red` | 48.2 s | 485 of 486 |
| blue, same size | `Blue` | 382.9 s | 0 |

That host run still ran the tower on a hit. The route now expands a picture's ids, positions and key from its plan
alone, looks the prompt up first, and runs the tower only when a picture row lies past the resume point. On the
NPU (`npu-hold`, amdxdna clean before and after), the same three requests:

| request | answer | time | cached |
|---|---|---|---|
| red | `Red` | 43.0 s | 0 |
| the same picture again | `Red` | 1.1 s | 485 of 486 (the tower is not run) |
| blue, same size | `Blue` | 55.5 s | 0 |

Prompts under the cache's `--prefix-min-gain` (128 tokens) are not saved at all, picture or not: a 448 × 448
picture with a short question is 93 rows.
- Speculative decoding is off for a request with pictures: its rotary positions are not the cache slots the
drafter counts.

`tests/npu_image_request_test.cpp` decodes crafted PPM and PNG data URLs to exactly their pixels, keeps request
order and names each refusal.

Three options exist to bisect a wrong answer against a reference run of the same prompt, and are how
the missing q/k norm above was found:

Expand Down
Loading
Loading