From 4aeb56a8eac94a8232a419f3fe19714121d3b2dd Mon Sep 17 00:00:00 2001 From: Brian Scanlan Date: Thu, 19 Nov 2026 13:18:53 +0000 Subject: [PATCH 1/2] Count a timed out spawn as one spawn error `hard_timeout` incremented `@consecutive_spawn_errors` for a child that was not ready. When the monitor reaps the killed child, `reap_worker` increments the counter again for the same failure. As a result, the monitor aborted after about half of `max_consecutive_spawn_errors` timeouts. Increment the counter in `hard_timeout` only for a child that never registered. The monitor never reaps such a child. `test_max_consecutive_spawn_timeout` passed only because of the double count: 3 timed out spawns counted as 6, which is more than 4. It now sets the limit to 2, so it still aborts after the first round of spawns. --- CHANGELOG.md | 1 + lib/pitchfork/http_server.rb | 3 +-- test/integration/test_boot.rb | 21 ++++++++++++++++++++- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe8a8af6..61252192 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - Fix `TTIN` during a graceful shutdown spawning new workers. - Prevent `USR1` from restarting the monitor during a graceful shutdown. - Honor shutdown signals while the server is starting. +- Fix `max_consecutive_spawn_errors` counting a timed out spawn twice. - Preserve worker spawn deadlines when the mold starts workers. # 0.18.4 diff --git a/lib/pitchfork/http_server.rb b/lib/pitchfork/http_server.rb index 9c5d8bc0..e6430a90 100644 --- a/lib/pitchfork/http_server.rb +++ b/lib/pitchfork/http_server.rb @@ -617,9 +617,8 @@ def murder_lazy_workers end def hard_timeout(child) - @consecutive_spawn_errors += 1 unless child.ready? - if child.pid.nil? # Not yet registered, likely never spawned + @consecutive_spawn_errors += 1 unless child.ready? logger.error "#{child.to_log} timed out during spawn, abandoning" @children.abandon(child) return diff --git a/test/integration/test_boot.rb b/test/integration/test_boot.rb index c30db58f..e7d62b12 100644 --- a/test/integration/test_boot.rb +++ b/test/integration/test_boot.rb @@ -117,7 +117,7 @@ def test_max_consecutive_spawn_timeout pid = spawn_server(app: APP, config: <<~RUBY) listen "#{addr}:#{port}" worker_processes 3 - max_consecutive_spawn_errors 4 + max_consecutive_spawn_errors 2 spawn_timeout 1 after_worker_fork do |_server, worker| sleep 20 # simulate a stuck worker @@ -128,6 +128,25 @@ def test_max_consecutive_spawn_timeout assert_exited(pid, 1) end + def test_max_consecutive_spawn_timeout_counts_each_timeout_once + addr, port = unused_port + + pid = spawn_server(app: APP, config: <<~RUBY) + listen "#{addr}:#{port}" + worker_processes 1 + max_consecutive_spawn_errors 1 + spawn_timeout 1 + after_worker_fork do |_server, worker| + sleep 20 + end + RUBY + + assert_stderr(/consecutive failures to spawn children, aborting/, timeout: 6) + assert_exited(pid, 1) + before_abort = read_stderr.split("consecutive failures to spawn children, aborting").first + assert_equal 2, before_abort.scan(/worker=0 gen=0.0 pid=\d+ timed out, killing/).size + end + def test_max_consecutive_spawn_errors addr, port = unused_port From b2da1e1173c19f1bce8e4b25ea81a242be082aec Mon Sep 17 00:00:00 2001 From: Brian Scanlan Date: Thu, 19 Nov 2026 22:46:44 +0000 Subject: [PATCH 2/2] Drop the ready? check for a child that never registered A child without a pid has not sent its spawn message, so it cannot be ready yet. --- lib/pitchfork/http_server.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pitchfork/http_server.rb b/lib/pitchfork/http_server.rb index e6430a90..55220857 100644 --- a/lib/pitchfork/http_server.rb +++ b/lib/pitchfork/http_server.rb @@ -618,7 +618,7 @@ def murder_lazy_workers def hard_timeout(child) if child.pid.nil? # Not yet registered, likely never spawned - @consecutive_spawn_errors += 1 unless child.ready? + @consecutive_spawn_errors += 1 logger.error "#{child.to_log} timed out during spawn, abandoning" @children.abandon(child) return