From ab08df5bea1c71f5bafd03883e731c6f85376ce2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20Mendon=C3=A7a=20Fran=C3=A7a?= Date: Wed, 18 Nov 2026 17:41:35 -0500 Subject: [PATCH 1/2] Preserve worker spawn deadlines in the mold The monitor sets a spawn deadline before it asks the mold to fork. The mold creates another Worker and resets the same shared state. This can clear the deadline and prevent the monitor from abandoning a slow spawn. Keep the shared state in the mold. Synchronize the timeout test so that it covers the race. --- .devcontainer/devcontainer-lock.json | 9 +++++++++ CHANGELOG.md | 2 ++ lib/pitchfork/http_server.rb | 8 ++++++-- lib/pitchfork/worker.rb | 10 +++++----- test/integration/test_boot.rb | 21 +++++++++++++++++++++ 5 files changed, 43 insertions(+), 7 deletions(-) create mode 100644 .devcontainer/devcontainer-lock.json diff --git a/.devcontainer/devcontainer-lock.json b/.devcontainer/devcontainer-lock.json new file mode 100644 index 00000000..de512861 --- /dev/null +++ b/.devcontainer/devcontainer-lock.json @@ -0,0 +1,9 @@ +{ + "features": { + "ghcr.io/devcontainers/features/github-cli:1": { + "version": "1.1.3", + "resolved": "ghcr.io/devcontainers/features/github-cli@sha256:bd7ab48a832228f633239277552c30b353867fef2e5b037e064b4e64f0b843f2", + "integrity": "sha256:bd7ab48a832228f633239277552c30b353867fef2e5b037e064b4e64f0b843f2" + } + } +} diff --git a/CHANGELOG.md b/CHANGELOG.md index cb9d6239..ec3f9ef0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Unreleased +- Preserve worker spawn deadlines when the mold starts workers. + # 0.18.4 - Avoid stale body source pointers when Ruby compacts strings. diff --git a/lib/pitchfork/http_server.rb b/lib/pitchfork/http_server.rb index a9a54bfd..bb8285b7 100644 --- a/lib/pitchfork/http_server.rb +++ b/lib/pitchfork/http_server.rb @@ -1266,6 +1266,7 @@ def mold_loop(mold) end def mold_handle_message(mold, message) + # The monitor initializes shared state and sets spawn deadlines before sending these messages. case message when false # no message, keep looping @@ -1273,7 +1274,7 @@ def mold_handle_message(mold, message) message.nrs.each do |nr| retries = 1 mold.update_deadline(@timeout) - worker = Worker.new(nr, generation: mold.generation, version: mold.version) + worker = Worker.new(nr, generation: mold.generation, version: mold.version, reset_state: false) begin spawn_worker(worker) rescue ForkFailure @@ -1292,7 +1293,10 @@ def mold_handle_message(mold, message) when Message::SpawnService retries = 1 begin - spawn_service(Pitchfork::Worker.new(nil, generation: mold.generation, version: mold.version, service: true)) + service = Pitchfork::Worker.new( + nil, generation: mold.generation, version: mold.version, service: true, reset_state: false + ) + spawn_service(service) rescue ForkFailure if retries > 0 @logger.fatal("#{mold.to_log} failed to spawn a service, retrying") diff --git a/lib/pitchfork/worker.rb b/lib/pitchfork/worker.rb index 2b1b9e59..7ed9c046 100644 --- a/lib/pitchfork/worker.rb +++ b/lib/pitchfork/worker.rb @@ -22,7 +22,7 @@ def load(data) end end - def initialize(nr, pid: nil, generation: 0, version: 0, service: false) + def initialize(nr, pid: nil, generation: 0, version: 0, service: false, reset_state: true) @nr = nr @pid = pid @generation = generation @@ -34,7 +34,7 @@ def initialize(nr, pid: nil, generation: 0, version: 0, service: false) @monitor = nil @exiting = false @requests_count = 0 - init_state + init_state(reset_state) end def init_with(data) @@ -342,13 +342,13 @@ def to_log private - def init_state + def init_state(reset_state) if nr @state_drop = SharedMemory.worker_state(@nr) - @state_drop.reset! + @state_drop.reset! if reset_state elsif service? @state_drop = SharedMemory.service_state - @state_drop.reset! + @state_drop.reset! if reset_state else promoted!(nil) end diff --git a/test/integration/test_boot.rb b/test/integration/test_boot.rb index 4bbdb813..c30db58f 100644 --- a/test/integration/test_boot.rb +++ b/test/integration/test_boot.rb @@ -211,6 +211,13 @@ module SlowSpawnWorkerPatch def spawn_worker(worker) @_spawn_worker_count ||= 0 @_spawn_worker_count += 1 + if @_spawn_worker_count == 1 + File.write("first-spawn-pending", "") + deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + 8 + until File.exist?("release-first-spawn") || Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline + sleep 0.01 + end + end super end @@ -226,9 +233,23 @@ def init_worker_process(worker) after_mold_fork do |server, _mold| server.extend(SlowSpawnWorkerPatch) end + + after_monitor_ready do |server| + def server.murder_lazy_workers + if children.workers_count > 0 && !@checked_first_spawn + deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + 5 + until File.exist?("first-spawn-pending") || Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline + sleep 0.01 + end + @checked_first_spawn = true + end + super + end + end RUBY assert_stderr(/worker=0 gen=0.0 timed out during spawn, abandoning/, timeout: 5) + File.write("release-first-spawn", "") assert_stderr(/worker=0 gen=0.0 pid=(\d+) spawned but slot is taken/, timeout: 5) abandoned_worker_pid = read_stderr.match(/worker=0 gen=0.0 pid=(\d+) spawned but slot is taken/)[1] From 4c366e4d239cf7671f101b1653b049986108739a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20Mendon=C3=A7a=20Fran=C3=A7a?= Date: Thu, 19 Nov 2026 10:59:18 -0500 Subject: [PATCH 2/2] Reset worker state explicitly in the monitor Worker initialization receives a flag to distinguish the monitor from the mold. The flag spreads process-specific behavior across the worker creation paths. Attach shared state during initialization, and reset it explicitly when the monitor creates a worker or service. The mold can then attach to the same state without resetting the spawn deadline. --- lib/pitchfork/http_server.rb | 14 +++++++------- lib/pitchfork/worker.rb | 28 +++++++++++++--------------- test/unit/test_worker.rb | 19 +++++++++++++++++++ 3 files changed, 39 insertions(+), 22 deletions(-) diff --git a/lib/pitchfork/http_server.rb b/lib/pitchfork/http_server.rb index bb8285b7..5056e098 100644 --- a/lib/pitchfork/http_server.rb +++ b/lib/pitchfork/http_server.rb @@ -847,8 +847,10 @@ def spawn_missing_workers generation = 0 end + # Only the monitor resets shared state; the mold attaches to the same slots. if @before_service_worker_ready && !@children.service service = Worker.new(nil, service: true, version: version, generation: generation) + service.reset_state! spawn_service_from_mold(service) end @@ -861,7 +863,9 @@ def spawn_missing_workers next end - missing_workers << Worker.new(worker_nr, version: version, generation: generation) + worker = Worker.new(worker_nr, version: version, generation: generation) + worker.reset_state! + missing_workers << worker end unless missing_workers.empty? @@ -1266,7 +1270,6 @@ def mold_loop(mold) end def mold_handle_message(mold, message) - # The monitor initializes shared state and sets spawn deadlines before sending these messages. case message when false # no message, keep looping @@ -1274,7 +1277,7 @@ def mold_handle_message(mold, message) message.nrs.each do |nr| retries = 1 mold.update_deadline(@timeout) - worker = Worker.new(nr, generation: mold.generation, version: mold.version, reset_state: false) + worker = Worker.new(nr, generation: mold.generation, version: mold.version) begin spawn_worker(worker) rescue ForkFailure @@ -1293,10 +1296,7 @@ def mold_handle_message(mold, message) when Message::SpawnService retries = 1 begin - service = Pitchfork::Worker.new( - nil, generation: mold.generation, version: mold.version, service: true, reset_state: false - ) - spawn_service(service) + spawn_service(Pitchfork::Worker.new(nil, generation: mold.generation, version: mold.version, service: true)) rescue ForkFailure if retries > 0 @logger.fatal("#{mold.to_log} failed to spawn a service, retrying") diff --git a/lib/pitchfork/worker.rb b/lib/pitchfork/worker.rb index 7ed9c046..4ec7d81f 100644 --- a/lib/pitchfork/worker.rb +++ b/lib/pitchfork/worker.rb @@ -22,7 +22,7 @@ def load(data) end end - def initialize(nr, pid: nil, generation: 0, version: 0, service: false, reset_state: true) + def initialize(nr, pid: nil, generation: 0, version: 0, service: false) @nr = nr @pid = pid @generation = generation @@ -34,7 +34,13 @@ def initialize(nr, pid: nil, generation: 0, version: 0, service: false, reset_st @monitor = nil @exiting = false @requests_count = 0 - init_state(reset_state) + if nr + @state_drop = SharedMemory.worker_state(@nr) + elsif service? + @state_drop = SharedMemory.service_state + else + promoted!(nil) + end end def init_with(data) @@ -306,6 +312,10 @@ def close # :nodoc: @to_io = nil end + def reset_state! + @state_drop.reset! + end + def reaped! close # Not ideal: All molds share two state_drops, so when reaping a mold @@ -314,7 +324,7 @@ def reaped! # unix sockets, we could consider assigning a unique page for each mold to solve # this problem. unless mold? - @state_drop.reset! + reset_state! end end @@ -342,18 +352,6 @@ def to_log private - def init_state(reset_state) - if nr - @state_drop = SharedMemory.worker_state(@nr) - @state_drop.reset! if reset_state - elsif service? - @state_drop = SharedMemory.service_state - @state_drop.reset! if reset_state - else - promoted!(nil) - end - end - def pipe=(socket) raise ArgumentError, "pipe can't be nil" unless socket Info.keep_io(socket) diff --git a/test/unit/test_worker.rb b/test/unit/test_worker.rb index c98f1bc8..a59b38cd 100644 --- a/test/unit/test_worker.rb +++ b/test/unit/test_worker.rb @@ -18,6 +18,25 @@ def test_create_many_workers end end + def test_new_worker_keeps_shared_state_until_reset + [[0, false], [nil, true]].each do |nr, service| + existing = Worker.new(nr, service: service) + existing.deadline = 123 + existing.ready = true + existing.busy = true + + worker = Worker.new(nr, service: service) + assert_equal 123, worker.deadline + assert worker.ready? + assert worker.busy? + + worker.reset_state! + assert_equal 0, existing.deadline + refute existing.ready? + refute existing.busy? + end + end + def test_shared_process worker = Worker.new(0) _, status = Process.waitpid2(fork { worker.deadline += 1; exit!(0) })