From 1dde6eb068e1d5b970fb16b7841b0f82c1aae916 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 19:42:46 -0400 Subject: [PATCH] forward-serve: a streamed reply is sent as it is generated, not after (Refs #1785) The NPU route built every SSE frame and sent them when the generation had ended, so time to first token was the whole request. A streaming request now generates on its own thread and hands frames to the connection thread through app/stream_bridge.h. Errors before the first frame keep their status codes; a client that leaves stops the generation. Co-Authored-By: Claude Fable 5.2 --- CMakeLists.txt | 8 +++ app/forward_serve.cpp | 92 +++++++++++++++++++++--- app/stream_bridge.h | 118 ++++++++++++++++++++++++++++++ docs/npu.md | 9 +++ tests/stream_bridge_test.cpp | 135 +++++++++++++++++++++++++++++++++++ 5 files changed, 354 insertions(+), 8 deletions(-) create mode 100644 app/stream_bridge.h create mode 100644 tests/stream_bridge_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 99ec81ca4..c4f3010d8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1131,6 +1131,14 @@ target_link_libraries(npu_stop_tokens_test PRIVATE nlohmann_json::nlohmann_json) add_test(NAME npu_stop_tokens COMMAND npu_stop_tokens_test) add_dependencies(npu_sampling_test npu_stop_tokens_test) +# The hand-off between the thread that generates a streamed reply and the one that writes it (app/stream_bridge.h, +# engine#1785: the NPU route sent every frame after the generation ended). Two threads and a clock; no device. +add_executable(stream_bridge_test tests/stream_bridge_test.cpp) +target_include_directories(stream_bridge_test PRIVATE ${CMAKE_SOURCE_DIR}/app) +target_link_libraries(stream_bridge_test PRIVATE Threads::Threads) +add_test(NAME stream_bridge COMMAND stream_bridge_test) +add_dependencies(npu_sampling_test stream_bridge_test) + # The plan.json loader (npu/plan/plan.h, engine #799 P1): schema, refusals and the decision fields that # replaced the filename probes. No model, no container, no device. add_executable(npu_elf_args_test tests/npu_elf_args_test.c npu/forward/src/q4nx_pack.c npu/forward/src/q4nx_dequant.c) diff --git a/app/forward_serve.cpp b/app/forward_serve.cpp index 1fb9db77e..5acc1c741 100644 --- a/app/forward_serve.cpp +++ b/app/forward_serve.cpp @@ -47,6 +47,7 @@ #include "npu_prompt_ids.h" #include "npu_sampling.h" #include "npu_stop_tokens.h" +#include "stream_bridge.h" #include "npu_systemone.h" #include "tokenizer.h" #include "../npu/decision.h" // the d3 readout head (npu/decision.h, issue #653) @@ -193,7 +194,10 @@ int json_int(const std::string& path, const char* key, int def) { return def; } -void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Response& res, bool chat) { +// `bridge` is where a streamed reply's frames go as they are produced (handle_generate below runs this on its own +// thread for a streaming request); null means the caller wants the whole response in `res`. +void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib::Response& res, bool chat, + npu::StreamBridge* bridge) { // One id per response, taken before any step: the dumps of two prompts in one serve must not share names. const int dump_req = e.dump_logits.empty() ? 0 : ++e.dump_seq; json body; @@ -287,7 +291,7 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res // Returns the finish_reason the request earned: "stop" for an end-of-sequence token, // a stop sequence or a grammar that allows nothing more, "length" when the token // budget ran out. - auto run = [&e, &req, prompt, prompt_ids, prompt_is_ids, max_tokens, chat, &opts, gram, &stats, dump_req]( + auto run = [&e, &req, prompt, prompt_ids, prompt_is_ids, max_tokens, chat, &opts, gram, &stats, dump_req, bridge]( const std::function&)>& on_text) { std::string finish = "stop"; // The tokenizer is const and keeps no per-call state, so it runs before the lock. A caller @@ -485,7 +489,8 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res bool stopped_at_eos = false, stopped_at_stop = false; int produced = 0; for (int n = 0; n < limit; n++) { - if (req.is_connection_closed()) break; // the client is gone: free the NPU + // the client is gone: free the NPU (the bridge knows first when a write to it failed) + if (req.is_connection_closed() || (bridge && bridge->cancelled())) break; if (gram) refresh_allowed(); // Raw distribution for a reference comparison: sample_token() takes logits by const // reference and the copy below happens before it runs, so what lands on disk is what @@ -657,7 +662,13 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res chunk_tpl["object"] = "chat.completion.chunk"; chunk_tpl["created"] = created; chunk_tpl["model"] = e.id; + // With a bridge, each frame leaves as soon as it exists (the connection thread is already writing them); + // without one they are collected and sent when the handler returns, as this route always did. auto chunks = std::make_shared>(); + auto send = [&](std::string frame) { + if (bridge) bridge->push(std::move(frame)); + else chunks->push_back(std::move(frame)); + }; // The same splitter as the non-streamed path, fed piece by piece, so a streamed reply // and a non-streamed one cannot disagree about what is reasoning and what is content. npu::AnswerStream stream(family, opts.tools_sent); @@ -675,7 +686,7 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res // carries no entries. if (!is_reasoning && opts.sample.logprobs && !lp.empty()) chunk["choices"][0]["logprobs"] = logprobs_json(lp); - chunks->push_back("data: " + chunk.dump(-1, ' ', false, json::error_handler_t::replace) + "\n\n"); + send("data: " + chunk.dump(-1, ' ', false, json::error_handler_t::replace) + "\n\n"); } return true; }; @@ -687,7 +698,7 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res json chunk = chunk_tpl; json delta = is_reasoning ? json{{"reasoning_content", text}} : json{{"content", text}}; chunk["choices"] = json::array({{{"index", 0}, {"delta", delta}, {"finish_reason", nullptr}}}); - chunks->push_back("data: " + chunk.dump(-1, ' ', false, json::error_handler_t::replace) + "\n\n"); + send("data: " + chunk.dump(-1, ' ', false, json::error_handler_t::replace) + "\n\n"); } if (!stream.calls().empty()) { // The calls arrive whole, in one chunk: OpenAI streams argument fragments, and a @@ -703,7 +714,7 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res json delta{{"tool_calls", calls}}; if (!chunk_tpl.contains("object") || chunk_tpl["object"].is_null()) chunk["object"] = "chat.completion.chunk"; chunk["choices"] = json::array({{{"index", 0}, {"delta", delta}, {"finish_reason", "tool_calls"}}}); - chunks->push_back("data: " + chunk.dump(-1, ' ', false, json::error_handler_t::replace) + "\n\n"); + send("data: " + chunk.dump(-1, ' ', false, json::error_handler_t::replace) + "\n\n"); } // OpenAI's stream ends with one choice that carries an empty delta and the reason the // generation ended; without it a streaming client cannot tell a `length` cut from a @@ -715,7 +726,7 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res chunk["choices"] = json::array({{{"index", 0}, {"delta", json::object()}, {"finish_reason", finish}}}); - chunks->push_back("data: " + chunk.dump(-1, ' ', false, json::error_handler_t::replace) + "\n\n"); + send("data: " + chunk.dump(-1, ' ', false, json::error_handler_t::replace) + "\n\n"); } } { @@ -723,7 +734,11 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res tail["choices"] = json::array(); tail["usage"] = usage_json(stats); tail["timings"] = timings_json(stats); - chunks->push_back("data: " + tail.dump() + "\n\n"); + send("data: " + tail.dump() + "\n\n"); + } + if (bridge) { + bridge->push("data: [DONE]\n\n"); + return; } auto cursor = std::make_shared(0); res.set_chunked_content_provider("text/event-stream", @@ -752,6 +767,67 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res } } +// The route's entry. A request that does not ask for a stream is answered in place. One that does is generated on +// its own thread, so its frames can be written while later tokens are still being produced (app/stream_bridge.h): +// this thread waits for the first frame or the end, and only then decides between an ordinary response -- the 400s +// and 500s a request can earn before it has produced anything keep their status codes -- and a stream. +void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Response& res, bool chat) { + bool wants_stream = false; + try { + const json body = json::parse(req.body); + wants_stream = body.is_object() && body.contains("stream") && body["stream"].is_boolean() && body["stream"].get(); + } catch (const std::exception&) { + // not JSON: the implementation answers that itself, without a stream + } + if (!wants_stream) { + handle_generate_impl(e, req, res, chat, nullptr); + return; + } + auto bridge = std::make_shared(); + auto early = std::make_shared(); // what the generation answers when it never starts a stream + // `req` outlives this handler: httplib keeps it until the response, content provider included, has been written, + // and the releaser below joins the thread before that. + auto worker = std::make_shared([&e, &req, chat, bridge, early]() { + try { + handle_generate_impl(e, req, *early, chat, bridge.get()); + } catch (const std::exception& ex) { + early->status = 500; + early->set_content(json{{"error", {{"message", ex.what()}}}}.dump(), "application/json"); + } catch (...) { + early->status = 500; + early->set_content(R"({"error":{"message":"generation failed"}})", "application/json"); + } + // A failure after the stream began cannot change the status any more: say so in the stream and end it. + // One that came before any frame pushes nothing, so the handler below still answers it as an ordinary error. + if (early->status >= 400 && bridge->started()) bridge->push("data: " + early->body + "\n\ndata: [DONE]\n\n"); + bridge->finish(); + }); + if (!bridge->wait_first()) { + worker->join(); + res.status = early->status > 0 ? early->status : 500; + res.set_content(early->body, early->has_header("Content-Type") ? early->get_header_value("Content-Type") + : std::string("application/json")); + return; + } + res.set_chunked_content_provider( + "text/event-stream", + [bridge](size_t, httplib::DataSink& sink) { + std::string frame; + while (bridge->next(frame)) { + if (!sink.write(frame.data(), frame.size())) { + bridge->cancel(); // the client went away: the generation stops at its next token + return false; + } + } + sink.done(); + return true; + }, + [bridge, worker](bool) { + bridge->cancel(); + if (worker->joinable()) worker->join(); + }); +} + // The per-question input limit of the decision route: the text path's own 32,768 tokens, or the // backbone's context when that is smaller. Either overflowing is the same failure to the model, so // the smaller one is the limit the request is checked against. diff --git a/app/stream_bridge.h b/app/stream_bridge.h new file mode 100644 index 000000000..cdca732c8 --- /dev/null +++ b/app/stream_bridge.h @@ -0,0 +1,118 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// stream_bridge.h -- hands a streamed reply from the thread that generates it to the thread that writes it. +// +// The NPU route used to build every server-sent-event frame of a streamed reply and send them all after the +// generation had ended, so a client's time to first token was the whole request (engine#1785: 14 chunks, every +// one arriving at 9.668 s of a 9.668 s reply). httplib writes a response body only after the handler returns, +// from a content provider it calls on the connection's thread, so frames can leave while tokens are still being +// produced only if the generation runs on another thread. This is the piece between the two. +// +// producer (the generation) consumer (the connection) +// push(frame) ... push(frame) wait_first() -> a frame is ready, or it ended without one +// finish() next(frame) -> the next frame in order; false when ended and drained +// cancelled() <- the client went away cancel() +// +// The decision that needs wait_first(): a request that fails before its first frame (a prompt over the context, +// a malformed field) must still be answered with its own status code, and a status cannot change once a stream +// has started. So the connection thread waits for the first frame OR the end, and only then chooses between +// "an ordinary error response" and "a stream". +// +// Frames are delivered in the order pushed, each exactly once. finish() may be called once; push() after it is +// ignored. Nothing here knows about HTTP. + +#pragma once + +#include +#include +#include +#include + +namespace onebit::npu { + +class StreamBridge { +public: + // Producer: queue one frame. Returns false when the consumer has cancelled, so the producer can stop early. + bool push(std::string frame) { + std::lock_guard lock(mu_); + if (finished_) return !cancelled_; + frames_.push_back(std::move(frame)); + ++pushed_; + cv_.notify_all(); + return !cancelled_; + } + + // Producer: no more frames. + void finish() { + std::lock_guard lock(mu_); + finished_ = true; + cv_.notify_all(); + } + + // Producer: has the consumer gone? + bool cancelled() const { + std::lock_guard lock(mu_); + return cancelled_; + } + + // Consumer: block until a frame has been pushed or the producer has finished. True when at least one frame + // was ever pushed (a stream should start), false when it ended with none (answer some other way). + bool wait_first() { + std::unique_lock lock(mu_); + cv_.wait(lock, [&] { return pushed_ > 0 || finished_; }); + return pushed_ > 0; + } + + // Consumer: the next frame, blocking while there is none and the producer is still going. False when the + // producer has finished and every frame has been taken, or after cancel(). + bool next(std::string& frame) { + std::unique_lock lock(mu_); + cv_.wait(lock, [&] { return !frames_.empty() || finished_ || cancelled_; }); + if (cancelled_ || frames_.empty()) return false; + frame = std::move(frames_.front()); + frames_.pop_front(); + return true; + } + + // Consumer: stop. The producer sees it at its next push() or cancelled(). + void cancel() { + std::lock_guard lock(mu_); + cancelled_ = true; + cv_.notify_all(); + } + + // Has any frame ever been pushed? (The producer asks this to decide whether a failure can still be an ordinary + // error response or has to be reported inside the stream.) + bool started() const { + std::lock_guard lock(mu_); + return pushed_ > 0; + } + + bool finished() const { + std::lock_guard lock(mu_); + return finished_; + } + +private: + mutable std::mutex mu_; + std::condition_variable cv_; + std::deque frames_; + size_t pushed_ = 0; + bool finished_ = false; + bool cancelled_ = false; +}; + +} // namespace onebit::npu diff --git a/docs/npu.md b/docs/npu.md index 2962883a6..76f584b16 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1520,6 +1520,15 @@ kernels (dense Qwen3: 0.6B, 1.7B and 4B): In Lemonade, the `onebit` recipe's `npu` backend serves the same directories (`--onebit npu`). The model directory holds the weights only; the kernels ship inside the package. +**Streaming.** With `"stream": true` the route sends each piece of the answer as it is produced. The first frame +leaves once the prompt has been processed and the first visible piece exists (a thinking model's opening markup is +held until it can be classified, so that is a token or two after the prompt is done), and the rest follow about +one per token. Before engine#1785 every frame was sent after the generation had ended, so a client's time to first +token was the whole request. A request that fails before producing anything (a prompt over the context, a +malformed field) still gets its own status code, not a 200 stream; one that fails later reports the error as a +last `data:` frame. A client that disconnects stops the generation at the next token, which frees the device for +the next request. The streamed reply and the non-streamed one are the same text. + ### FastFlowLM's NPU2 drops: the families the route does not serve The daily census-watch (engine#588) prints one line per new or changed repo in FastFlowLM's HF diff --git a/tests/stream_bridge_test.cpp b/tests/stream_bridge_test.cpp new file mode 100644 index 000000000..87b4934a5 --- /dev/null +++ b/tests/stream_bridge_test.cpp @@ -0,0 +1,135 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// stream_bridge_test: app/stream_bridge.h, the hand-off between the thread that generates a streamed reply and the +// thread that writes it (engine#1785). No server, no model, no device: two threads and a clock. + +#include "stream_bridge.h" + +#include +#include +#include +#include +#include +#include + +using onebit::npu::StreamBridge; +using clk = std::chrono::steady_clock; + +static int failures = 0; +static void check(bool ok, const std::string& what) { + if (!ok) { + std::printf("FAIL %s\n", what.c_str()); + ++failures; + } +} +static double ms_since(clk::time_point t) { return std::chrono::duration(clk::now() - t).count(); } + +int main() { + // 1. The point of the bridge: a frame is readable when it is pushed, not when the producer finishes. + { + StreamBridge b; + const auto t0 = clk::now(); + std::thread producer([&] { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + b.push("one"); + std::this_thread::sleep_for(std::chrono::milliseconds(400)); + b.push("two"); + b.finish(); + }); + const bool started = b.wait_first(); + const double first_ms = ms_since(t0); + std::string f; + std::vector got; + std::vector at; + while (b.next(f)) { + got.push_back(f); + at.push_back(ms_since(t0)); + } + producer.join(); + check(started, "wait_first reports a stream once a frame exists"); + check(first_ms < 300, "wait_first returns at the first frame, not at the end (" + std::to_string(first_ms) + " ms of ~450)"); + check(got == std::vector({"one", "two"}), "frames arrive in order, each once"); + check(at.size() == 2 && at[0] < 300 && at[1] >= 400, "the first frame is taken well before the second exists"); + check(b.finished() && b.started(), "finished and started are both true at the end"); + } + // 2. A producer that ends without a frame: no stream, so the caller can still answer with a status code. + { + StreamBridge b; + std::thread producer([&] { + std::this_thread::sleep_for(std::chrono::milliseconds(30)); + b.finish(); + }); + check(!b.wait_first(), "wait_first is false when the producer ended with no frame"); + std::string f; + check(!b.next(f) && !b.started(), "and there is nothing to read"); + producer.join(); + } + // 3. Frames pushed before anyone waits are kept, and the end is seen only after they are drained. + { + StreamBridge b; + b.push("a"); + b.push("b"); + b.finish(); + b.push("late"); // after finish: ignored + std::string f; + check(b.wait_first() && b.next(f) && f == "a" && b.next(f) && f == "b" && !b.next(f), + "queued frames survive finish() and a push after finish() is dropped"); + } + // 4. cancel(): the consumer stops, a blocked next() returns, and the producer is told. + { + StreamBridge b; + std::atomic producer_saw{false}; + std::thread producer([&] { + b.push("x"); + for (int i = 0; i < 200 && !b.cancelled(); i++) std::this_thread::sleep_for(std::chrono::milliseconds(5)); + producer_saw = b.cancelled(); + check(!b.push("after cancel"), "push reports the cancellation to the producer"); + b.finish(); + }); + std::string f; + check(b.next(f) && f == "x", "the frame before the cancel is delivered"); + std::thread canceller([&] { + std::this_thread::sleep_for(std::chrono::milliseconds(40)); + b.cancel(); + }); + const auto t0 = clk::now(); + check(!b.next(f), "next() returns false once cancelled"); + check(ms_since(t0) < 500, "and a next() that was blocked is released by the cancel"); + canceller.join(); + producer.join(); + check(producer_saw.load(), "the producer sees cancelled()"); + } + // 5. Many frames from a fast producer: none lost, none repeated, order kept. + { + StreamBridge b; + const int n = 20000; + std::thread producer([&] { + for (int i = 0; i < n; i++) b.push(std::to_string(i)); + b.finish(); + }); + std::string f; + int expect = 0; + bool ordered = true; + while (b.next(f)) { + ordered = ordered && f == std::to_string(expect); + ++expect; + } + producer.join(); + check(ordered && expect == n, "20000 frames cross in order with none lost (" + std::to_string(expect) + ")"); + } + if (failures == 0) std::printf("stream_bridge: ok\n"); + return failures == 0 ? 0 : 1; +}