From be56f926aa2cdeb3a6195228f2ab08344b1bc29c Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 09:16:42 -0400 Subject: [PATCH 1/5] serve: a backend whose port was taken after free_port() is relaunched on a fresh port, and serve names the cause (engine#2537) Co-authored-by: bong-water-water-bong Co-Authored-By: Claude Opus 5.5 --- app/serve.cpp | 42 +++++++++++++++++++++++++++++++++++++++++- tests/fake_backend.py | 7 +++++++ tests/hybrid_route.sh | 11 +++++++++++ 3 files changed, 59 insertions(+), 1 deletion(-) diff --git a/app/serve.cpp b/app/serve.cpp index d5553887b..0a2ca2e60 100644 --- a/app/serve.cpp +++ b/app/serve.cpp @@ -922,6 +922,21 @@ void relay(int port, const httplib::Request& req, httplib::Response& res) { res.set_content(r->body, r->get_header_value("Content-Type").empty() ? "application/json" : r->get_header_value("Content-Type")); } +// engine#2537: the line a backend prints when its listening port is already bound. Python's socketserver +// ("OSError: [Errno 98] Address already in use") and llama-server ("couldn't bind HTTP server socket"). +inline bool port_in_use_line(const std::string& line) { + return line.find("Address already in use") != std::string::npos || + line.find("couldn't bind HTTP server socket") != std::string::npos; +} + +// Rewrite the value after "--port" in a backend's argv; the launch built it with the port serve first picked. +inline void set_port_arg(std::vector& argv, int port) { + for (size_t k = 0; k + 1 < argv.size(); ++k) + if (argv[k] == "--port") { argv[k + 1] = std::to_string(port); return; } +} + +constexpr int kPortTakenRetries = 2; + struct Launch { std::string device; int port = 0; @@ -2385,8 +2400,14 @@ int serve_child(const Options& given) { Lifecycle lifecycle(id); std::mutex wedged_mu; std::string wedged; // set by the watchdog: why the backend is taken down - auto backend_line = [&metrics, &placement_worst, &wedged_mu, &wedged](const std::string& line) { + // engine#2537: free_port() releases the port before the backend binds it, and a backend takes a moment before it + // binds (a model load; tests/fake_backend.py sleeps 0.5 s), so under load another process can be handed the same + // port in between. The backend then dies with EADDRINUSE; this records that, so serve names it and relaunches the + // backend on a fresh port instead of reporting "did not become ready". + std::atomic backend_port_taken{false}; + auto backend_line = [&metrics, &placement_worst, &wedged_mu, &wedged, &backend_port_taken](const std::string& line) { metrics.backend_line(line); + if (port_in_use_line(line)) backend_port_taken = true; if (device_lost_line(line)) { std::lock_guard lock(wedged_mu); if (wedged.empty()) wedged = "the backend lost its GPU: " + line.substr(0, 200); @@ -2760,8 +2781,27 @@ int serve_child(const Options& given) { // Launch::device is "hrx", or "hrx reranking" for a role or RAG server: the device is the first word auto on_device = [](const Launch& b) { return b.device.substr(0, b.device.find(' ')); }; for (size_t i = 0; i < children.size(); i++) { + // engine#2537: a backend that died because its port was taken is relaunched on a fresh one, twice at most. Only that + // cause is retried: any other failure to start still fails serve, below. + for (int retry = 1; !laya && i < backends.size() && retry <= kPortTakenRetries; ++retry) { + backend_port_taken = false; + if (children[i]->wait_ready(std::chrono::seconds(600), g_stop) || g_stop) break; + if (children[i]->alive() || !backend_port_taken) break; + const int taken = backends[i].port; + backends[i].port = free_port(); + set_port_arg(backends[i].argv, backends[i].port); + std::fprintf(stderr, "1bit serve: backend %zu could not bind port %d (address already in use: another process took it " + "after serve picked it); relaunching on port %d (%d of %d)\n", + i, taken, backends[i].port, retry, kPortTakenRetries); + children[i] = std::make_unique(backends[i].argv, backends[i].env, backends[i].port, backends[i].ready_path, + backend_line, backends[i].env_override); + } if (!children[i]->wait_ready(std::chrono::seconds(600), g_stop)) { if (g_stop) return 0; + if (backend_port_taken && !children[i]->alive()) { + std::fprintf(stderr, "1bit serve: backend %zu could not bind port %d: address already in use\n", i, children[i]->port()); + return 1; + } // a strict HRX backend (app/hrx_strict.h) that refused a node said which; serve says it again if (const std::string why = metrics.strict_refusal(); !why.empty()) std::fprintf(stderr, "1bit serve: %s\n", strict_refusal_message(why).c_str()); diff --git a/tests/fake_backend.py b/tests/fake_backend.py index 1dd68e116..030d76fd2 100755 --- a/tests/fake_backend.py +++ b/tests/fake_backend.py @@ -228,6 +228,13 @@ def do_POST(self): "ggml_hrx: CPU fallback: GET_ROWS x2, SOFTPLUS x1"): print(line, file=sys.stderr, flush=True) time.sleep(0.5) # a real backend takes a moment to load; serve must answer 503 meanwhile +if os.environ.get("FAKE_BACKEND_PORT_TAKEN_ONCE"): + # engine#2537: the first launch finds its port taken (another process bound it between serve's free_port() and this + # bind), as socketserver reports it; the marker makes the relaunch on serve's fresh port bind normally. + marker = os.environ["FAKE_BACKEND_PORT_TAKEN_ONCE"] + if not os.path.exists(marker): + open(marker, "w").close() + raise OSError(98, "Address already in use") if os.environ.get("FAKE_BACKEND_IMPOSTOR"): # A process serve did not start answers on the port instead: this one detaches a server # (writing its pid to $FAKE_BACKEND_IMPOSTOR) and stays alive without listening itself. diff --git a/tests/hybrid_route.sh b/tests/hybrid_route.sh index 1034e67ae..c8ffd0d87 100755 --- a/tests/hybrid_route.sh +++ b/tests/hybrid_route.sh @@ -177,6 +177,17 @@ signal.signal(signal.SIGTERM, term) time.sleep(30) PY chmod +x "$scratch/finishing_provider.py" +# engine#2537: a backend whose port was taken after serve picked it (free_port() releases the port, the backend binds it a +# moment later) is relaunched on a fresh port, and serve says why, instead of "backend 0 did not become ready". +check "serve starts when the backend's first port is taken" \ + 'start "$scratch/t.log" FAKE_BACKEND_PORT_TAKEN_ONCE="$scratch/port-taken.marker"' +check "serve names the taken port and relaunches" 'grep -q "could not bind port [0-9]* (address already in use.*relaunching on port [0-9]* (1 of 2)" "$scratch/t.log"' +reply=$(curl -s "$api/v1/chat/completions" -H 'Content-Type: application/json' \ + -d '{"model": "hybrid-model", "messages": [{"role": "user", "content": "What is the capital of France?"}]}') +check "the relaunched backend answers" '[[ "$reply" == *Paris* ]]' +check "no did-not-become-ready for a taken port" '! grep -q "did not become ready" "$scratch/t.log"' +stop + check "serve starts with a provider that finishes after its deadline" \ 'start "$scratch/f.log" ONEBIT_HYBRID_PREFILL="$scratch/finishing_provider.py" ONEBIT_HYBRID_PREFILL_TIMEOUT_S=1' reply=$(curl -s "$api/v1/chat/completions" -H 'Content-Type: application/json' \ From 064c633a6756f54967199bd5062c72e665dc28d5 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 09:23:45 -0400 Subject: [PATCH 2/5] serve: clear the port-taken flag only before a relaunch, so an EADDRINUSE printed while serve waits on an earlier backend is kept (engine#2537) Co-authored-by: bong-water-water-bong Co-Authored-By: Claude Opus 5.5 --- app/serve.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/serve.cpp b/app/serve.cpp index 0a2ca2e60..e68b8dc88 100644 --- a/app/serve.cpp +++ b/app/serve.cpp @@ -2784,7 +2784,6 @@ int serve_child(const Options& given) { // engine#2537: a backend that died because its port was taken is relaunched on a fresh one, twice at most. Only that // cause is retried: any other failure to start still fails serve, below. for (int retry = 1; !laya && i < backends.size() && retry <= kPortTakenRetries; ++retry) { - backend_port_taken = false; if (children[i]->wait_ready(std::chrono::seconds(600), g_stop) || g_stop) break; if (children[i]->alive() || !backend_port_taken) break; const int taken = backends[i].port; @@ -2793,6 +2792,9 @@ int serve_child(const Options& given) { std::fprintf(stderr, "1bit serve: backend %zu could not bind port %d (address already in use: another process took it " "after serve picked it); relaunching on port %d (%d of %d)\n", i, taken, backends[i].port, retry, kPortTakenRetries); + // cleared only here, before the relaunch: a backend can print its EADDRINUSE while serve is still waiting on an + // earlier backend, and clearing the flag before this backend's wait would drop that line + backend_port_taken = false; children[i] = std::make_unique(backends[i].argv, backends[i].env, backends[i].port, backends[i].ready_path, backend_line, backends[i].env_override); } From d1b357b23735ca6419a64c032e8c9d7374426ac0 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 09:33:55 -0400 Subject: [PATCH 3/5] serve: the give-up line counts the relaunches; hybrid_route asserts the relaunch is on a fresh port (engine#2537 review) Co-authored-by: bong-water-water-bong Co-Authored-By: Claude Opus 5.5 --- app/serve.cpp | 3 ++- tests/hybrid_route.sh | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/app/serve.cpp b/app/serve.cpp index e68b8dc88..2ca27df58 100644 --- a/app/serve.cpp +++ b/app/serve.cpp @@ -2801,7 +2801,8 @@ int serve_child(const Options& given) { if (!children[i]->wait_ready(std::chrono::seconds(600), g_stop)) { if (g_stop) return 0; if (backend_port_taken && !children[i]->alive()) { - std::fprintf(stderr, "1bit serve: backend %zu could not bind port %d: address already in use\n", i, children[i]->port()); + std::fprintf(stderr, "1bit serve: backend %zu could not bind port %d: address already in use, after %d relaunches " + "on fresh ports; giving up\n", i, children[i]->port(), kPortTakenRetries); return 1; } // a strict HRX backend (app/hrx_strict.h) that refused a node said which; serve says it again diff --git a/tests/hybrid_route.sh b/tests/hybrid_route.sh index c8ffd0d87..fcdfc0ccf 100755 --- a/tests/hybrid_route.sh +++ b/tests/hybrid_route.sh @@ -185,6 +185,10 @@ check "serve names the taken port and relaunches" 'grep -q "could not bind port reply=$(curl -s "$api/v1/chat/completions" -H 'Content-Type: application/json' \ -d '{"model": "hybrid-model", "messages": [{"role": "user", "content": "What is the capital of France?"}]}') check "the relaunched backend answers" '[[ "$reply" == *Paris* ]]' +# the relaunch is on a different port from the taken one: a change that reintroduced a fixed backend port fails here +taken=$(grep -o "could not bind port [0-9]*" "$scratch/t.log" | head -1 | grep -o "[0-9]*$") +relaunched=$(grep -o "relaunching on port [0-9]*" "$scratch/t.log" | head -1 | grep -o "[0-9]*$") +check "the relaunch uses a fresh port ($taken -> $relaunched)" '[ -n "$taken" ] && [ -n "$relaunched" ] && [ "$taken" != "$relaunched" ]' check "no did-not-become-ready for a taken port" '! grep -q "did not become ready" "$scratch/t.log"' stop From b63d63842638589e76d6a7ec406d7d2705cfded7 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 09:45:49 -0400 Subject: [PATCH 4/5] serve: the port-taken flag is per backend, so one backend's taken port cannot relaunch another (engine#2537 review) Co-authored-by: bong-water-water-bong Co-Authored-By: Claude Opus 5.5 --- app/serve.cpp | 33 ++++++++++++++++++++++----------- tests/hybrid_route.sh | 2 ++ 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/app/serve.cpp b/app/serve.cpp index 2ca27df58..bb7a11e9e 100644 --- a/app/serve.cpp +++ b/app/serve.cpp @@ -2403,11 +2403,11 @@ int serve_child(const Options& given) { // engine#2537: free_port() releases the port before the backend binds it, and a backend takes a moment before it // binds (a model load; tests/fake_backend.py sleeps 0.5 s), so under load another process can be handed the same // port in between. The backend then dies with EADDRINUSE; this records that, so serve names it and relaunches the - // backend on a fresh port instead of reporting "did not become ready". - std::atomic backend_port_taken{false}; - auto backend_line = [&metrics, &placement_worst, &wedged_mu, &wedged, &backend_port_taken](const std::string& line) { + // backend on a fresh port instead of reporting "did not become ready". One flag per backend: a single shared one said that + // *some* backend printed EADDRINUSE, so a second backend's taken port could relaunch a first one that failed otherwise. + std::vector>> port_taken; + auto backend_line = [&metrics, &placement_worst, &wedged_mu, &wedged](const std::string& line) { metrics.backend_line(line); - if (port_in_use_line(line)) backend_port_taken = true; if (device_lost_line(line)) { std::lock_guard lock(wedged_mu); if (wedged.empty()) wedged = "the backend lost its GPU: " + line.substr(0, 200); @@ -2769,9 +2769,20 @@ int serve_child(const Options& given) { ::sigaction(SIGTERM, &sa, nullptr); ::sigaction(SIGINT, &sa, nullptr); #endif - for (const auto& b : laya ? std::vector{} : backends) { - std::fprintf(stderr, "1bit serve: %s on %s (%s)\n", id.c_str(), b.device.c_str(), b.argv[0].c_str()); - children.push_back(std::make_unique(b.argv, b.env, b.port, b.ready_path, backend_line, b.env_override)); + for (size_t k = 0; k < backends.size(); ++k) port_taken.push_back(std::make_unique>(false)); + // backend i's line callback: the shared one, plus backend i's own port-taken flag (engine#2537) + auto line_for = [&backend_line, &port_taken](size_t i) { + return [&backend_line, flag = port_taken[i].get()](const std::string& l) { + backend_line(l); + if (port_in_use_line(l)) *flag = true; + }; + }; + if (!laya) { + for (size_t k = 0; k < backends.size(); ++k) { + const auto& b = backends[k]; + std::fprintf(stderr, "1bit serve: %s on %s (%s)\n", id.c_str(), b.device.c_str(), b.argv[0].c_str()); + children.push_back(std::make_unique(b.argv, b.env, b.port, b.ready_path, line_for(k), b.env_override)); + } } for (const auto& b : rag) { if (b.port == 0) continue; // in-process: the encoder is this process (app/embed_serve.h) @@ -2785,7 +2796,7 @@ int serve_child(const Options& given) { // cause is retried: any other failure to start still fails serve, below. for (int retry = 1; !laya && i < backends.size() && retry <= kPortTakenRetries; ++retry) { if (children[i]->wait_ready(std::chrono::seconds(600), g_stop) || g_stop) break; - if (children[i]->alive() || !backend_port_taken) break; + if (children[i]->alive() || !*port_taken[i]) break; const int taken = backends[i].port; backends[i].port = free_port(); set_port_arg(backends[i].argv, backends[i].port); @@ -2794,13 +2805,13 @@ int serve_child(const Options& given) { i, taken, backends[i].port, retry, kPortTakenRetries); // cleared only here, before the relaunch: a backend can print its EADDRINUSE while serve is still waiting on an // earlier backend, and clearing the flag before this backend's wait would drop that line - backend_port_taken = false; + *port_taken[i] = false; children[i] = std::make_unique(backends[i].argv, backends[i].env, backends[i].port, backends[i].ready_path, - backend_line, backends[i].env_override); + line_for(i), backends[i].env_override); } if (!children[i]->wait_ready(std::chrono::seconds(600), g_stop)) { if (g_stop) return 0; - if (backend_port_taken && !children[i]->alive()) { + if (!laya && i < port_taken.size() && *port_taken[i] && !children[i]->alive()) { std::fprintf(stderr, "1bit serve: backend %zu could not bind port %d: address already in use, after %d relaunches " "on fresh ports; giving up\n", i, children[i]->port(), kPortTakenRetries); return 1; diff --git a/tests/hybrid_route.sh b/tests/hybrid_route.sh index fcdfc0ccf..665b839d0 100755 --- a/tests/hybrid_route.sh +++ b/tests/hybrid_route.sh @@ -177,6 +177,8 @@ signal.signal(signal.SIGTERM, term) time.sleep(30) PY chmod +x "$scratch/finishing_provider.py" +# engine#2537 (multi-backend case unpinned: outside --laya, serve launches one backend, and --laya's backends and RAG children are +# not retried, so two retry-eligible backends cannot be started here; the flag is per backend all the same) # engine#2537: a backend whose port was taken after serve picked it (free_port() releases the port, the backend binds it a # moment later) is relaunched on a fresh port, and serve says why, instead of "backend 0 did not become ready". check "serve starts when the backend's first port is taken" \ From 9ced29fff79dd96c7d8eb131023e0d3f390ae58b Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 10:09:16 -0400 Subject: [PATCH 5/5] tests: the #2537 comments say how serve picks a port without naming free_port, which route_port_lock_test reads as a script binding one Co-authored-by: bong-water-water-bong Co-Authored-By: Claude Opus 5.5 --- tests/fake_backend.py | 2 +- tests/hybrid_route.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/fake_backend.py b/tests/fake_backend.py index 030d76fd2..76a701f67 100755 --- a/tests/fake_backend.py +++ b/tests/fake_backend.py @@ -229,7 +229,7 @@ def do_POST(self): print(line, file=sys.stderr, flush=True) time.sleep(0.5) # a real backend takes a moment to load; serve must answer 503 meanwhile if os.environ.get("FAKE_BACKEND_PORT_TAKEN_ONCE"): - # engine#2537: the first launch finds its port taken (another process bound it between serve's free_port() and this + # engine#2537: the first launch finds its port taken (another process bound it between serve picking it and this # bind), as socketserver reports it; the marker makes the relaunch on serve's fresh port bind normally. marker = os.environ["FAKE_BACKEND_PORT_TAKEN_ONCE"] if not os.path.exists(marker): diff --git a/tests/hybrid_route.sh b/tests/hybrid_route.sh index 665b839d0..0989ad45e 100755 --- a/tests/hybrid_route.sh +++ b/tests/hybrid_route.sh @@ -179,7 +179,7 @@ PY chmod +x "$scratch/finishing_provider.py" # engine#2537 (multi-backend case unpinned: outside --laya, serve launches one backend, and --laya's backends and RAG children are # not retried, so two retry-eligible backends cannot be started here; the flag is per backend all the same) -# engine#2537: a backend whose port was taken after serve picked it (free_port() releases the port, the backend binds it a +# engine#2537: a backend whose port was taken after serve picked it (serve releases the port it picked, the backend binds it a # moment later) is relaunched on a fresh port, and serve says why, instead of "backend 0 did not become ready". check "serve starts when the backend's first port is taken" \ 'start "$scratch/t.log" FAKE_BACKEND_PORT_TAKEN_ONCE="$scratch/port-taken.marker"'