diff --git a/app/serve.cpp b/app/serve.cpp index d5553887b..bb7a11e9e 100644 --- a/app/serve.cpp +++ b/app/serve.cpp @@ -922,6 +922,21 @@ void relay(int port, const httplib::Request& req, httplib::Response& res) { res.set_content(r->body, r->get_header_value("Content-Type").empty() ? "application/json" : r->get_header_value("Content-Type")); } +// engine#2537: the line a backend prints when its listening port is already bound. Python's socketserver +// ("OSError: [Errno 98] Address already in use") and llama-server ("couldn't bind HTTP server socket"). +inline bool port_in_use_line(const std::string& line) { + return line.find("Address already in use") != std::string::npos || + line.find("couldn't bind HTTP server socket") != std::string::npos; +} + +// Rewrite the value after "--port" in a backend's argv; the launch built it with the port serve first picked. +inline void set_port_arg(std::vector& argv, int port) { + for (size_t k = 0; k + 1 < argv.size(); ++k) + if (argv[k] == "--port") { argv[k + 1] = std::to_string(port); return; } +} + +constexpr int kPortTakenRetries = 2; + struct Launch { std::string device; int port = 0; @@ -2385,6 +2400,12 @@ int serve_child(const Options& given) { Lifecycle lifecycle(id); std::mutex wedged_mu; std::string wedged; // set by the watchdog: why the backend is taken down + // engine#2537: free_port() releases the port before the backend binds it, and a backend takes a moment before it + // binds (a model load; tests/fake_backend.py sleeps 0.5 s), so under load another process can be handed the same + // port in between. The backend then dies with EADDRINUSE; this records that, so serve names it and relaunches the + // backend on a fresh port instead of reporting "did not become ready". One flag per backend: a single shared one said that + // *some* backend printed EADDRINUSE, so a second backend's taken port could relaunch a first one that failed otherwise. + std::vector>> port_taken; auto backend_line = [&metrics, &placement_worst, &wedged_mu, &wedged](const std::string& line) { metrics.backend_line(line); if (device_lost_line(line)) { @@ -2748,9 +2769,20 @@ int serve_child(const Options& given) { ::sigaction(SIGTERM, &sa, nullptr); ::sigaction(SIGINT, &sa, nullptr); #endif - for (const auto& b : laya ? std::vector{} : backends) { - std::fprintf(stderr, "1bit serve: %s on %s (%s)\n", id.c_str(), b.device.c_str(), b.argv[0].c_str()); - children.push_back(std::make_unique(b.argv, b.env, b.port, b.ready_path, backend_line, b.env_override)); + for (size_t k = 0; k < backends.size(); ++k) port_taken.push_back(std::make_unique>(false)); + // backend i's line callback: the shared one, plus backend i's own port-taken flag (engine#2537) + auto line_for = [&backend_line, &port_taken](size_t i) { + return [&backend_line, flag = port_taken[i].get()](const std::string& l) { + backend_line(l); + if (port_in_use_line(l)) *flag = true; + }; + }; + if (!laya) { + for (size_t k = 0; k < backends.size(); ++k) { + const auto& b = backends[k]; + std::fprintf(stderr, "1bit serve: %s on %s (%s)\n", id.c_str(), b.device.c_str(), b.argv[0].c_str()); + children.push_back(std::make_unique(b.argv, b.env, b.port, b.ready_path, line_for(k), b.env_override)); + } } for (const auto& b : rag) { if (b.port == 0) continue; // in-process: the encoder is this process (app/embed_serve.h) @@ -2760,8 +2792,30 @@ int serve_child(const Options& given) { // Launch::device is "hrx", or "hrx reranking" for a role or RAG server: the device is the first word auto on_device = [](const Launch& b) { return b.device.substr(0, b.device.find(' ')); }; for (size_t i = 0; i < children.size(); i++) { + // engine#2537: a backend that died because its port was taken is relaunched on a fresh one, twice at most. Only that + // cause is retried: any other failure to start still fails serve, below. + for (int retry = 1; !laya && i < backends.size() && retry <= kPortTakenRetries; ++retry) { + if (children[i]->wait_ready(std::chrono::seconds(600), g_stop) || g_stop) break; + if (children[i]->alive() || !*port_taken[i]) break; + const int taken = backends[i].port; + backends[i].port = free_port(); + set_port_arg(backends[i].argv, backends[i].port); + std::fprintf(stderr, "1bit serve: backend %zu could not bind port %d (address already in use: another process took it " + "after serve picked it); relaunching on port %d (%d of %d)\n", + i, taken, backends[i].port, retry, kPortTakenRetries); + // cleared only here, before the relaunch: a backend can print its EADDRINUSE while serve is still waiting on an + // earlier backend, and clearing the flag before this backend's wait would drop that line + *port_taken[i] = false; + children[i] = std::make_unique(backends[i].argv, backends[i].env, backends[i].port, backends[i].ready_path, + line_for(i), backends[i].env_override); + } if (!children[i]->wait_ready(std::chrono::seconds(600), g_stop)) { if (g_stop) return 0; + if (!laya && i < port_taken.size() && *port_taken[i] && !children[i]->alive()) { + std::fprintf(stderr, "1bit serve: backend %zu could not bind port %d: address already in use, after %d relaunches " + "on fresh ports; giving up\n", i, children[i]->port(), kPortTakenRetries); + return 1; + } // a strict HRX backend (app/hrx_strict.h) that refused a node said which; serve says it again if (const std::string why = metrics.strict_refusal(); !why.empty()) std::fprintf(stderr, "1bit serve: %s\n", strict_refusal_message(why).c_str()); diff --git a/tests/fake_backend.py b/tests/fake_backend.py index 1dd68e116..76a701f67 100755 --- a/tests/fake_backend.py +++ b/tests/fake_backend.py @@ -228,6 +228,13 @@ def do_POST(self): "ggml_hrx: CPU fallback: GET_ROWS x2, SOFTPLUS x1"): print(line, file=sys.stderr, flush=True) time.sleep(0.5) # a real backend takes a moment to load; serve must answer 503 meanwhile +if os.environ.get("FAKE_BACKEND_PORT_TAKEN_ONCE"): + # engine#2537: the first launch finds its port taken (another process bound it between serve picking it and this + # bind), as socketserver reports it; the marker makes the relaunch on serve's fresh port bind normally. + marker = os.environ["FAKE_BACKEND_PORT_TAKEN_ONCE"] + if not os.path.exists(marker): + open(marker, "w").close() + raise OSError(98, "Address already in use") if os.environ.get("FAKE_BACKEND_IMPOSTOR"): # A process serve did not start answers on the port instead: this one detaches a server # (writing its pid to $FAKE_BACKEND_IMPOSTOR) and stays alive without listening itself. diff --git a/tests/hybrid_route.sh b/tests/hybrid_route.sh index 1034e67ae..0989ad45e 100755 --- a/tests/hybrid_route.sh +++ b/tests/hybrid_route.sh @@ -177,6 +177,23 @@ signal.signal(signal.SIGTERM, term) time.sleep(30) PY chmod +x "$scratch/finishing_provider.py" +# engine#2537 (multi-backend case unpinned: outside --laya, serve launches one backend, and --laya's backends and RAG children are +# not retried, so two retry-eligible backends cannot be started here; the flag is per backend all the same) +# engine#2537: a backend whose port was taken after serve picked it (serve releases the port it picked, the backend binds it a +# moment later) is relaunched on a fresh port, and serve says why, instead of "backend 0 did not become ready". +check "serve starts when the backend's first port is taken" \ + 'start "$scratch/t.log" FAKE_BACKEND_PORT_TAKEN_ONCE="$scratch/port-taken.marker"' +check "serve names the taken port and relaunches" 'grep -q "could not bind port [0-9]* (address already in use.*relaunching on port [0-9]* (1 of 2)" "$scratch/t.log"' +reply=$(curl -s "$api/v1/chat/completions" -H 'Content-Type: application/json' \ + -d '{"model": "hybrid-model", "messages": [{"role": "user", "content": "What is the capital of France?"}]}') +check "the relaunched backend answers" '[[ "$reply" == *Paris* ]]' +# the relaunch is on a different port from the taken one: a change that reintroduced a fixed backend port fails here +taken=$(grep -o "could not bind port [0-9]*" "$scratch/t.log" | head -1 | grep -o "[0-9]*$") +relaunched=$(grep -o "relaunching on port [0-9]*" "$scratch/t.log" | head -1 | grep -o "[0-9]*$") +check "the relaunch uses a fresh port ($taken -> $relaunched)" '[ -n "$taken" ] && [ -n "$relaunched" ] && [ "$taken" != "$relaunched" ]' +check "no did-not-become-ready for a taken port" '! grep -q "did not become ready" "$scratch/t.log"' +stop + check "serve starts with a provider that finishes after its deadline" \ 'start "$scratch/f.log" ONEBIT_HYBRID_PREFILL="$scratch/finishing_provider.py" ONEBIT_HYBRID_PREFILL_TIMEOUT_S=1' reply=$(curl -s "$api/v1/chat/completions" -H 'Content-Type: application/json' \