Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1131,6 +1131,14 @@ target_link_libraries(npu_stop_tokens_test PRIVATE nlohmann_json::nlohmann_json)
add_test(NAME npu_stop_tokens COMMAND npu_stop_tokens_test)
add_dependencies(npu_sampling_test npu_stop_tokens_test)

# The hand-off between the thread that generates a streamed reply and the one that writes it (app/stream_bridge.h,
# engine#1785: the NPU route sent every frame after the generation ended). Two threads and a clock; no device.
add_executable(stream_bridge_test tests/stream_bridge_test.cpp)
target_include_directories(stream_bridge_test PRIVATE ${CMAKE_SOURCE_DIR}/app)
target_link_libraries(stream_bridge_test PRIVATE Threads::Threads)
add_test(NAME stream_bridge COMMAND stream_bridge_test)
add_dependencies(npu_sampling_test stream_bridge_test)

# The plan.json loader (npu/plan/plan.h, engine #799 P1): schema, refusals and the decision fields that
# replaced the filename probes. No model, no container, no device.
add_executable(npu_elf_args_test tests/npu_elf_args_test.c npu/forward/src/q4nx_pack.c npu/forward/src/q4nx_dequant.c)
Expand Down
92 changes: 84 additions & 8 deletions app/forward_serve.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
#include "npu_prompt_ids.h"
#include "npu_sampling.h"
#include "npu_stop_tokens.h"
#include "stream_bridge.h"
#include "npu_systemone.h"
#include "tokenizer.h"
#include "../npu/decision.h" // the d3 readout head (npu/decision.h, issue #653)
Expand Down Expand Up @@ -193,7 +194,10 @@ int json_int(const std::string& path, const char* key, int def) {
return def;
}

void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Response& res, bool chat) {
// `bridge` is where a streamed reply's frames go as they are produced (handle_generate below runs this on its own
// thread for a streaming request); null means the caller wants the whole response in `res`.
void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib::Response& res, bool chat,
npu::StreamBridge* bridge) {
// One id per response, taken before any step: the dumps of two prompts in one serve must not share names.
const int dump_req = e.dump_logits.empty() ? 0 : ++e.dump_seq;
json body;
Expand Down Expand Up @@ -287,7 +291,7 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res
// Returns the finish_reason the request earned: "stop" for an end-of-sequence token,
// a stop sequence or a grammar that allows nothing more, "length" when the token
// budget ran out.
auto run = [&e, &req, prompt, prompt_ids, prompt_is_ids, max_tokens, chat, &opts, gram, &stats, dump_req](
auto run = [&e, &req, prompt, prompt_ids, prompt_is_ids, max_tokens, chat, &opts, gram, &stats, dump_req, bridge](
const std::function<bool(const std::string&, const std::vector<Emitted>&)>& on_text) {
std::string finish = "stop";
// The tokenizer is const and keeps no per-call state, so it runs before the lock. A caller
Expand Down Expand Up @@ -485,7 +489,8 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res
bool stopped_at_eos = false, stopped_at_stop = false;
int produced = 0;
for (int n = 0; n < limit; n++) {
if (req.is_connection_closed()) break; // the client is gone: free the NPU
// the client is gone: free the NPU (the bridge knows first when a write to it failed)
if (req.is_connection_closed() || (bridge && bridge->cancelled())) break;
if (gram) refresh_allowed();
// Raw distribution for a reference comparison: sample_token() takes logits by const
// reference and the copy below happens before it runs, so what lands on disk is what
Expand Down Expand Up @@ -657,7 +662,13 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res
chunk_tpl["object"] = "chat.completion.chunk";
chunk_tpl["created"] = created;
chunk_tpl["model"] = e.id;
// With a bridge, each frame leaves as soon as it exists (the connection thread is already writing them);
// without one they are collected and sent when the handler returns, as this route always did.
auto chunks = std::make_shared<std::vector<std::string>>();
auto send = [&](std::string frame) {
if (bridge) bridge->push(std::move(frame));
else chunks->push_back(std::move(frame));
};
// The same splitter as the non-streamed path, fed piece by piece, so a streamed reply
// and a non-streamed one cannot disagree about what is reasoning and what is content.
npu::AnswerStream stream(family, opts.tools_sent);
Expand All @@ -675,7 +686,7 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res
// carries no entries.
if (!is_reasoning && opts.sample.logprobs && !lp.empty())
chunk["choices"][0]["logprobs"] = logprobs_json(lp);
chunks->push_back("data: " + chunk.dump(-1, ' ', false, json::error_handler_t::replace) + "\n\n");
send("data: " + chunk.dump(-1, ' ', false, json::error_handler_t::replace) + "\n\n");
}
return true;
};
Expand All @@ -687,7 +698,7 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res
json chunk = chunk_tpl;
json delta = is_reasoning ? json{{"reasoning_content", text}} : json{{"content", text}};
chunk["choices"] = json::array({{{"index", 0}, {"delta", delta}, {"finish_reason", nullptr}}});
chunks->push_back("data: " + chunk.dump(-1, ' ', false, json::error_handler_t::replace) + "\n\n");
send("data: " + chunk.dump(-1, ' ', false, json::error_handler_t::replace) + "\n\n");
}
if (!stream.calls().empty()) {
// The calls arrive whole, in one chunk: OpenAI streams argument fragments, and a
Expand All @@ -703,7 +714,7 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res
json delta{{"tool_calls", calls}};
if (!chunk_tpl.contains("object") || chunk_tpl["object"].is_null()) chunk["object"] = "chat.completion.chunk";
chunk["choices"] = json::array({{{"index", 0}, {"delta", delta}, {"finish_reason", "tool_calls"}}});
chunks->push_back("data: " + chunk.dump(-1, ' ', false, json::error_handler_t::replace) + "\n\n");
send("data: " + chunk.dump(-1, ' ', false, json::error_handler_t::replace) + "\n\n");
}
// OpenAI's stream ends with one choice that carries an empty delta and the reason the
// generation ended; without it a streaming client cannot tell a `length` cut from a
Expand All @@ -715,15 +726,19 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res
chunk["choices"] = json::array({{{"index", 0},
{"delta", json::object()},
{"finish_reason", finish}}});
chunks->push_back("data: " + chunk.dump(-1, ' ', false, json::error_handler_t::replace) + "\n\n");
send("data: " + chunk.dump(-1, ' ', false, json::error_handler_t::replace) + "\n\n");
}
}
{
json tail = chunk_tpl;
tail["choices"] = json::array();
tail["usage"] = usage_json(stats);
tail["timings"] = timings_json(stats);
chunks->push_back("data: " + tail.dump() + "\n\n");
send("data: " + tail.dump() + "\n\n");
}
if (bridge) {
bridge->push("data: [DONE]\n\n");
return;
}
auto cursor = std::make_shared<size_t>(0);
res.set_chunked_content_provider("text/event-stream",
Expand Down Expand Up @@ -752,6 +767,67 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res
}
}

// The route's entry. A request that does not ask for a stream is answered in place. One that does is generated on
// its own thread, so its frames can be written while later tokens are still being produced (app/stream_bridge.h):
// this thread waits for the first frame or the end, and only then decides between an ordinary response -- the 400s
// and 500s a request can earn before it has produced anything keep their status codes -- and a stream.
void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Response& res, bool chat) {
bool wants_stream = false;
try {
const json body = json::parse(req.body);
wants_stream = body.is_object() && body.contains("stream") && body["stream"].is_boolean() && body["stream"].get<bool>();
} catch (const std::exception&) {
// not JSON: the implementation answers that itself, without a stream
}
if (!wants_stream) {
handle_generate_impl(e, req, res, chat, nullptr);
return;
}
auto bridge = std::make_shared<npu::StreamBridge>();
auto early = std::make_shared<httplib::Response>(); // what the generation answers when it never starts a stream
// `req` outlives this handler: httplib keeps it until the response, content provider included, has been written,
// and the releaser below joins the thread before that.
auto worker = std::make_shared<std::thread>([&e, &req, chat, bridge, early]() {
try {
handle_generate_impl(e, req, *early, chat, bridge.get());
} catch (const std::exception& ex) {
early->status = 500;
early->set_content(json{{"error", {{"message", ex.what()}}}}.dump(), "application/json");
} catch (...) {
early->status = 500;
early->set_content(R"({"error":{"message":"generation failed"}})", "application/json");
}
// A failure after the stream began cannot change the status any more: say so in the stream and end it.
// One that came before any frame pushes nothing, so the handler below still answers it as an ordinary error.
if (early->status >= 400 && bridge->started()) bridge->push("data: " + early->body + "\n\ndata: [DONE]\n\n");
bridge->finish();
});
if (!bridge->wait_first()) {
worker->join();
res.status = early->status > 0 ? early->status : 500;
res.set_content(early->body, early->has_header("Content-Type") ? early->get_header_value("Content-Type")
: std::string("application/json"));
return;
}
res.set_chunked_content_provider(
"text/event-stream",
[bridge](size_t, httplib::DataSink& sink) {
std::string frame;
while (bridge->next(frame)) {
if (!sink.write(frame.data(), frame.size())) {
bridge->cancel(); // the client went away: the generation stops at its next token
return false;
}
}
sink.done();
return true;
},
[bridge, worker](bool) {
bridge->cancel();
if (worker->joinable()) worker->join();
});
}

// The per-question input limit of the decision route: the text path's own 32,768 tokens, or the
// backbone's context when that is smaller. Either overflowing is the same failure to the model, so
// the smaller one is the limit the request is checked against.
Expand Down
118 changes: 118 additions & 0 deletions app/stream_bridge.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
// Copyright 2026 bong-water-water-bong
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// stream_bridge.h -- hands a streamed reply from the thread that generates it to the thread that writes it.
//
// The NPU route used to build every server-sent-event frame of a streamed reply and send them all after the
// generation had ended, so a client's time to first token was the whole request (engine#1785: 14 chunks, every
// one arriving at 9.668 s of a 9.668 s reply). httplib writes a response body only after the handler returns,
// from a content provider it calls on the connection's thread, so frames can leave while tokens are still being
// produced only if the generation runs on another thread. This is the piece between the two.
//
// producer (the generation) consumer (the connection)
// push(frame) ... push(frame) wait_first() -> a frame is ready, or it ended without one
// finish() next(frame) -> the next frame in order; false when ended and drained
// cancelled() <- the client went away cancel()
//
// The decision that needs wait_first(): a request that fails before its first frame (a prompt over the context,
// a malformed field) must still be answered with its own status code, and a status cannot change once a stream
// has started. So the connection thread waits for the first frame OR the end, and only then chooses between
// "an ordinary error response" and "a stream".
//
// Frames are delivered in the order pushed, each exactly once. finish() may be called once; push() after it is
// ignored. Nothing here knows about HTTP.

#pragma once

#include <condition_variable>
#include <deque>
#include <mutex>
#include <string>

namespace onebit::npu {

class StreamBridge {
public:
// Producer: queue one frame. Returns false when the consumer has cancelled, so the producer can stop early.
bool push(std::string frame) {
std::lock_guard<std::mutex> lock(mu_);
if (finished_) return !cancelled_;
frames_.push_back(std::move(frame));
++pushed_;
cv_.notify_all();
return !cancelled_;
}

// Producer: no more frames.
void finish() {
std::lock_guard<std::mutex> lock(mu_);
finished_ = true;
cv_.notify_all();
}

// Producer: has the consumer gone?
bool cancelled() const {
std::lock_guard<std::mutex> lock(mu_);
return cancelled_;
}

// Consumer: block until a frame has been pushed or the producer has finished. True when at least one frame
// was ever pushed (a stream should start), false when it ended with none (answer some other way).
bool wait_first() {
std::unique_lock<std::mutex> lock(mu_);
cv_.wait(lock, [&] { return pushed_ > 0 || finished_; });
return pushed_ > 0;
}

// Consumer: the next frame, blocking while there is none and the producer is still going. False when the
// producer has finished and every frame has been taken, or after cancel().
bool next(std::string& frame) {
std::unique_lock<std::mutex> lock(mu_);
cv_.wait(lock, [&] { return !frames_.empty() || finished_ || cancelled_; });
if (cancelled_ || frames_.empty()) return false;
frame = std::move(frames_.front());
frames_.pop_front();
return true;
}

// Consumer: stop. The producer sees it at its next push() or cancelled().
void cancel() {
std::lock_guard<std::mutex> lock(mu_);
cancelled_ = true;
cv_.notify_all();
}

// Has any frame ever been pushed? (The producer asks this to decide whether a failure can still be an ordinary
// error response or has to be reported inside the stream.)
bool started() const {
std::lock_guard<std::mutex> lock(mu_);
return pushed_ > 0;
}

bool finished() const {
std::lock_guard<std::mutex> lock(mu_);
return finished_;
}

private:
mutable std::mutex mu_;
std::condition_variable cv_;
std::deque<std::string> frames_;
size_t pushed_ = 0;
bool finished_ = false;
bool cancelled_ = false;
};

} // namespace onebit::npu
9 changes: 9 additions & 0 deletions docs/npu.md
Original file line number Diff line number Diff line change
Expand Up @@ -1520,6 +1520,15 @@ kernels (dense Qwen3: 0.6B, 1.7B and 4B):
In Lemonade, the `onebit` recipe's `npu` backend serves the same directories (`--onebit npu`).
The model directory holds the weights only; the kernels ship inside the package.

**Streaming.** With `"stream": true` the route sends each piece of the answer as it is produced. The first frame
leaves once the prompt has been processed and the first visible piece exists (a thinking model's opening markup is
held until it can be classified, so that is a token or two after the prompt is done), and the rest follow about
one per token. Before engine#1785 every frame was sent after the generation had ended, so a client's time to first
token was the whole request. A request that fails before producing anything (a prompt over the context, a
malformed field) still gets its own status code, not a 200 stream; one that fails later reports the error as a
last `data:` frame. A client that disconnects stops the generation at the next token, which frees the device for
the next request. The streamed reply and the non-streamed one are the same text.

### FastFlowLM's NPU2 drops: the families the route does not serve

The daily census-watch (engine#588) prints one line per new or changed repo in FastFlowLM's HF
Expand Down
Loading
Loading