diff --git a/app/forward_serve.cpp b/app/forward_serve.cpp index 84c5c15f8..271f1e4b4 100644 --- a/app/forward_serve.cpp +++ b/app/forward_serve.cpp @@ -587,7 +587,7 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res } return true; }; - run(emit_pieces); + const std::string finish = run(emit_pieces); { std::vector> tail; stream.finish(tail); @@ -613,6 +613,18 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res chunk["choices"] = json::array({{{"index", 0}, {"delta", delta}, {"finish_reason", "tool_calls"}}}); chunks->push_back("data: " + chunk.dump(-1, ' ', false, json::error_handler_t::replace) + "\n\n"); } + // OpenAI's stream ends with one choice that carries an empty delta and the reason the + // generation ended; without it a streaming client cannot tell a `length` cut from a + // natural `stop` (the per-delta chunks all say null, and the usage tail has no + // choices at all). The tool-call chunk above already reports `tool_calls`, so this is + // sent exactly once, and only for the text path. + if (stream.calls().empty()) { + json chunk = chunk_tpl; + chunk["choices"] = json::array({{{"index", 0}, + {"delta", json::object()}, + {"finish_reason", finish}}}); + chunks->push_back("data: " + chunk.dump(-1, ' ', false, json::error_handler_t::replace) + "\n\n"); + } } { json tail = chunk_tpl;