From a32f20e67a3a70ceba8c3547f337eacaee0e8266 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Fri, 13 Nov 2026 06:23:00 -0400 Subject: [PATCH 1/2] pm_route: a dead port is bound-and-held, not freeport()ed main CI at 50b68e76 failed 'pm_route' on the arm "an unreachable executor: the PM says so and still answers". The cause is the arm's setup, not the feature: `dead_port=$(freeport)` binds port 0, reads the number and CLOSES the socket, so between that close and the PM's connect the kernel is free to hand the same port to anything else on the box -- and on a busy self-hosted runner something does, which makes the "unreachable" executor reachable and the assertion fail. Caught it in the act: a port from freeport() is re-bindable immediately after it returns. `deadport()` holds a socket that is bound and never `listen()`ed, for the life of the test. That gives both properties the arm needs and freeport() only gave the first: connect(127.0.0.1, port) -> ECONNREFUSED (a bound socket with no backlog refuses) bind(127.0.0.1, port) -> EADDRINUSE (nobody else can take it and 'revive' the executor) Measured: 0 of 20 trials could take the dead port (freeport() could be taken every time), and the whole `ctest -R '^pm_route$'` row passes with the change. Not a behaviour change: only the port the *test* declares dead, and the socket is killed in the existing cleanup trap with the other helpers. --- tests/pm_route.sh | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/pm_route.sh b/tests/pm_route.sh index c3e676ec2..2e24edca6 100755 --- a/tests/pm_route.sh +++ b/tests/pm_route.sh @@ -48,6 +48,21 @@ trap cleanup EXIT fail=0 check() { if eval "$2"; then echo "ok $1"; else echo "FAIL $1"; fail=1; fi; } freeport() { python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1])'; } +# A port that refuses connections AND cannot be taken by another process: bound, never `listen()`ed, and +# held open for the life of the test. `freeport()` above closes its socket, so the kernel is free to hand +# that port to anything else on the box -- and on a busy runner something does, which makes the +# "unreachable executor" arm reachable and fails it (main CI, 50b68e76). Here connect() gets ECONNREFUSED +# (a bound socket with no backlog refuses) and bind() gets EADDRINUSE, so the arm tests what it says. +deadport() { + local pf="$scratch/dead.port" + python3 -c 'import socket,sys,time +s = socket.socket(); s.bind(("127.0.0.1", 0)) +open(sys.argv[1], "w").write(str(s.getsockname()[1])) +time.sleep(600)' "$pf" & + pids+=($!) + for _ in $(seq 1 200); do [ -s "$pf" ] && break; sleep 0.02; done + cat "$pf" +} jq() { python3 -c 'import json,sys; d=json.load(sys.stdin); print(eval(sys.argv[1]))' "$1"; } # jq EXPR over stdin JSON as d lem_port=$(freeport) @@ -116,7 +131,7 @@ check " its reached the expert" '[[ "$(tail -1 "$scratch/le check " and the answer came back" '[[ "$(echo "$reply" | jq "d[\"choices\"][0][\"message\"][\"content\"]")" == *EXPERT_ANSWER* ]]' # 6. Lemonade not running -dead_port=$(freeport) +dead_port=$(deadport) serve dead --lemonade-url "http://127.0.0.1:$dead_port" || fail=1 code=$(curl -s -o "$scratch/dead.json" -w '%{http_code}' "$api/v1/chat/completions" -H 'Content-Type: application/json' \ -d '{"model": "ZAYA1-8B-PM", "messages": [{"role": "user", "content": "write a C function that reverses a string"}]}') From ef3af6a59cfde304295da51f6deb802db2e5a5a6 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Fri, 13 Nov 2026 07:02:00 -0400 Subject: [PATCH 2/2] pm_route: both dead-port arms, and a sleeper that does not hold the pipe Two CHANGES on #2355, both right: 1. **The wrong call site.** `dead_port=$(freeport)` appears twice -- line 134 (the Lemonade-unreachable arm) and line 182 (the *executor*-unreachable arm, which is the one that failed on main). My replacement took the first match, so the arm I was fixing kept `freeport()`. Both use `deadport()` now. 2. **The helper blocked for the whole sleep.** A backgrounded child inherits the stdout of $( ), so the substitution waits for EOF -- i.e. for `time.sleep(600)` to end. Measured the shape directly: 4018 ms for a 4 s sleeper with the fd inherited, against 4023 ms for the file-passing variant I wrote; both hang. The child is now redirected to /dev/null, and the assignment measures **26 ms**. That second one is why my own verification missed it: I grepped the run for 'tests passed' and never read the duration, and the row was taking 625.60 sec -- inside ctest's 600 s timeout only by luck on that machine. Re-measured after the fix: **ctest -R '^pm_route$' -> 100% passed in 25.33 sec**. Still the same primitive, verified again: connect -> ECONNREFUSED in 0.1 ms, and bind by another socket -> EADDRINUSE, so the executor is unreachable and nothing can revive it. --- tests/pm_route.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/pm_route.sh b/tests/pm_route.sh index 2e24edca6..ccd12a197 100755 --- a/tests/pm_route.sh +++ b/tests/pm_route.sh @@ -55,10 +55,12 @@ freeport() { python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", # (a bound socket with no backlog refuses) and bind() gets EADDRINUSE, so the arm tests what it says. deadport() { local pf="$scratch/dead.port" + # >/dev/null: the child must not inherit $( )'s stdout, or the substitution waits for the sleep to + # end -- measured: 4015 ms for a 4 s sleeper with the fd inherited, 27 ms detached (review of #2355). python3 -c 'import socket,sys,time s = socket.socket(); s.bind(("127.0.0.1", 0)) open(sys.argv[1], "w").write(str(s.getsockname()[1])) -time.sleep(600)' "$pf" & +time.sleep(600)' "$pf" >/dev/null 2>&1 & pids+=($!) for _ in $(seq 1 200); do [ -s "$pf" ] && break; sleep 0.02; done cat "$pf" @@ -179,7 +181,7 @@ check "a refusal from the executor reaches the client" '[[ "$(echo "$reply" | jq check " pm.spawns says ok false" '[ "$(echo "$reply" | jq "d[\"pm\"][\"spawns\"][0][\"ok\"]")" = False ]' check " and the PM still answered (HTTP 200)" '[ -n "$reply" ]' -dead_port=$(freeport) +dead_port=$(deadport) PM_CALL_FORM=spawn serve spawndead --lemonade-url "$lemonade" --pm-spawn-url "http://127.0.0.1:$dead_port" || fail=1 reply=$(ask) check "an unreachable executor: the PM says so and still answers" '[[ "$(echo "$reply" | jq "d[\"choices\"][0][\"message\"][\"content\"]")" == *"did not answer"* ]]'