From 5d58afee586271a2c033c1210a25f0d348b26b80 Mon Sep 17 00:00:00 2001 From: sprooty Date: Sat, 8 Aug 2026 08:02:33 +0000 Subject: [PATCH] #207: the pause test asserts claiming stopped, not how fast a host is Third build of the deployment image failed here: FAILED tests/test_fleet.py::test_pausing_a_project_stops_claiming AssertionError: work continued after the project was paused assert 2 == 1 Nothing was wrong. The test paused a project, slept 100ms, sampled, slept 150ms and asserted the sample had not moved -- so an item already in flight when the pause landed, finishing inside that window on a loaded machine, read as "the pause did not work". The build host is a Docker-in-Docker worker under contention, which is exactly the machine that exposes it. #207 already named the fix: the property worth protecting is that pausing stops *claiming*, and the old assertion measured how fast an in-flight item finishes instead. So: - `settle()` waits for completions to stop moving rather than assuming a window, which tolerates a slow host instead of asserting against it; - `queue.claim()` is called directly while paused and must return None. That is the promise, tested with no timing assumption at all; - the queue must still hold pending work, so a drained backlog can no longer pass by proving nothing. Ran six times on a contended machine without a flake. The resume half already waited on a condition and is unchanged. --- tests/test_fleet.py | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/tests/test_fleet.py b/tests/test_fleet.py index 8923d18..94f0de6 100644 --- a/tests/test_fleet.py +++ b/tests/test_fleet.py @@ -69,6 +69,27 @@ def wait_for(predicate, timeout: float = 5.0) -> bool: # type: ignore[no-untype return False +def settle(sample, timeout: float = 20.0, quiet: float = 0.2): # type: ignore[no-untyped-def] + """Wait until `sample()` stops changing, and return the settled value. + + A condition, not a sleep. Work already in flight when a project is paused + still finishes, and how long that takes is a fact about the host -- so + waiting for movement to stop tolerates a slow machine, where a fixed + window silently becomes an assertion about CPU contention (#207). + """ + deadline = time.time() + timeout + last = sample() + stable_since = time.time() + while time.time() < deadline: + time.sleep(0.02) + current = sample() + if current != last: + last, stable_since = current, time.time() + elif time.time() - stable_since >= quiet: + return last + return last + + # ------------------------------------------------------------- starting @@ -486,10 +507,16 @@ def test_pausing_a_project_stops_claiming_without_a_restart(tmp_path: Path) -> N fleet.start("a") assert wait_for(lambda: len(seen) >= 1) q.set_control("paused", reason="testing", project_id="a") - time.sleep(0.1) - settled = len(seen) - time.sleep(0.15) - assert len(seen) == settled, "work continued after the project was paused" + + # An item already in flight finishes; that is not work *claimed* after + # the pause, and how long it takes belongs to the host. So settle, + # then assert the property the pause actually promises. + settled = settle(lambda: len(seen)) + assert q.claim("probe", project_id="a") is None, "a paused project handed out work" + assert q.counts("a").get(PENDING, 0) > 0, ( + "the backlog drained before the pause, so this proved nothing" + ) + assert settle(lambda: len(seen)) == settled, "claiming continued while paused" q.set_control(RUNNING, project_id="a") assert wait_for(lambda: len(seen) > settled), "resuming needed a restart"