From fce787950dcac044cd809c2efa754cbf06598f49 Mon Sep 17 00:00:00 2001 From: Brian Scanlan Date: Thu, 19 Nov 2026 16:49:09 +0000 Subject: [PATCH] Ignore MoldReady from a mold that was already reaped If a mold exits right after sending `MoldReady`, `reap_all_workers` can remove it before the monitor reads the message. `Children#update` then falls back to a lookup by `nr`, which `MoldReady` doesn't have, and raises `NoMethodError`. Had it returned nil, the monitor would have failed on `nil.to_log` instead. In the main loop the error is rescued and logged. During shutdown it escapes `stop`, so the monitor exits without killing the children that are still running. Only fall back to `nr` for messages that carry one, and have the monitor log and ignore `MoldReady` for a mold it no longer tracks. The integration test pauses the monitor with SIGSTOP so the mold always exits before its `MoldReady` is read. --- CHANGELOG.md | 1 + lib/pitchfork/children.rb | 2 +- lib/pitchfork/http_server.rb | 5 +++- test/integration/test_reforking.rb | 42 ++++++++++++++++++++++++++++++ test/unit/test_children.rb | 11 ++++++++ 5 files changed, 59 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec3f9ef0..0ccdd645 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Unreleased - Preserve worker spawn deadlines when the mold starts workers. +- Fix the monitor raising `NoMethodError` when a mold exits before its `MoldReady` message is handled. # 0.18.4 diff --git a/lib/pitchfork/children.rb b/lib/pitchfork/children.rb index 4280d46d..b693ff45 100644 --- a/lib/pitchfork/children.rb +++ b/lib/pitchfork/children.rb @@ -97,7 +97,7 @@ def update(message) return service end - if child = @children[message.pid] || (message.nr && @workers[message.nr]) + if child = @children[message.pid] || (message.respond_to?(:nr) && message.nr && @workers[message.nr]) if child.pid && child.pid != message.pid # An abandoned spawn may still come up late: never rebind a slot that already has a process. return diff --git a/lib/pitchfork/http_server.rb b/lib/pitchfork/http_server.rb index 5056e098..b56acd4c 100644 --- a/lib/pitchfork/http_server.rb +++ b/lib/pitchfork/http_server.rb @@ -430,7 +430,10 @@ def monitor_handle_message(message) message.pipe.close end when Message::MoldReady - new_mold = @children.update(message) + unless new_mold = @children.update(message) + logger.info("mold pid=#{message.pid} ready but already reaped, ignoring") + return + end # current_mold may not be new_mold if there was a race # between refork and restart. diff --git a/test/integration/test_reforking.rb b/test/integration/test_reforking.rb index 939908b8..4473a919 100644 --- a/test/integration/test_reforking.rb +++ b/test/integration/test_reforking.rb @@ -153,6 +153,48 @@ def test_exiting_mold assert_clean_shutdown(pid) end + def test_mold_reaped_before_ready_message + addr, port = unused_port + + pid = spawn_server(app: File.join(ROOT, "test/integration/env.ru"), config: <<~CONFIG) + listen "#{addr}:#{port}" + worker_processes 2 + refork_after [5, 5] + + monitor_pid = Process.pid + after_mold_fork do |_server, mold| + if mold.generation > 0 && !File.exist?("exited-once.txt") + File.write("exited-once.txt", "1") + sleep 0.01 until File.read("stderr.log").include?("pid=\#{Process.pid} spawned") + # Pause the monitor so it reaps this mold before reading its MoldReady message. + Process.kill(:STOP, monitor_pid) + def mold.finish_promotion(control_socket) + super + $stderr.puts "[mold exiting]" + exit!(0) + end + end + end + CONFIG + + assert_healthy("http://#{addr}:#{port}") + assert_stderr(/worker=1 gen=0.0 pid=\d+ ready/, timeout: 5) + + 7.times do + assert_equal true, healthy?("http://#{addr}:#{port}") + end + + assert_stderr("[mold exiting]", timeout: 3) + mold_pid = Integer(read_stderr[/mold gen=0\.1 pid=(\d+) spawned/, 1]) + sleep 0.01 until File.read("/proc/#{mold_pid}/stat").rpartition(")").last.split.first == "Z" + Process.kill(:CONT, pid) + + assert_stderr("mold pid=#{mold_pid} ready but already reaped, ignoring") + refute_stderr(/monitor loop error/) + + assert_clean_shutdown(pid) + end + def test_stuck_mold addr, port = unused_port diff --git a/test/unit/test_children.rb b/test/unit/test_children.rb index 3dc5d8de..edb91cd6 100644 --- a/test/unit/test_children.rb +++ b/test/unit/test_children.rb @@ -214,6 +214,17 @@ def test_reap_pending_mold assert_nil @children.reap(mold.pid) end + def test_mold_ready_after_reap + pipe = IO.pipe.last + @children.update(Message::MoldSpawned.new(nil, 42, 1, 0, pipe)) + @children.reap(42) + + assert_nil @children.update(Message::MoldReady.new(42)) + assert_nil @children.mold + assert_equal [], @children.molds + refute_predicate @children, :pending_promotion? + end + def test_dump_load_pending_service @children.register_service(Worker.new(nil, service: true, version: 2, generation: 3)) @children = assert_roundtrip(@children)