From 9978667ac77f7cc2a525e6a431f7fe9faf556965 Mon Sep 17 00:00:00 2001 From: Brian Scanlan Date: Thu, 19 Nov 2026 13:15:15 +0000 Subject: [PATCH 1/6] Don't resume respawning workers on TTIN The `TTIN` handler set `@respawn = true`. In Unicorn, this line undid `WINCH`. Pitchfork has no `WINCH`, so `@respawn` is false only during boot and during shutdown. A `TTIN` during a graceful shutdown made the monitor spawn new workers, and these workers could outlive the monitor. --- CHANGELOG.md | 1 + lib/pitchfork/http_server.rb | 1 - test/integration/test_shutdown.rb | 44 +++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 test/integration/test_shutdown.rb diff --git a/CHANGELOG.md b/CHANGELOG.md index ec3f9ef0..61daa3ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ # Unreleased +- Fix `TTIN` during a graceful shutdown spawning new workers. - Preserve worker spawn deadlines when the mold starts workers. # 0.18.4 diff --git a/lib/pitchfork/http_server.rb b/lib/pitchfork/http_server.rb index 5056e098..c9bcf575 100644 --- a/lib/pitchfork/http_server.rb +++ b/lib/pitchfork/http_server.rb @@ -411,7 +411,6 @@ def monitor_handle_message(message) logger.error "Can't trigger a refork as the server is shutting down" end when :TTIN - @respawn = true self.worker_processes += 1 when :TTOU self.worker_processes -= 1 if self.worker_processes > 0 diff --git a/test/integration/test_shutdown.rb b/test/integration/test_shutdown.rb new file mode 100644 index 00000000..19947660 --- /dev/null +++ b/test/integration/test_shutdown.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true +require 'integration_test_helper' + +class TestShutdown < Pitchfork::IntegrationTest + SLOW_APP = <<~'RUBY' + run lambda { |env| + if (seconds = env["QUERY_STRING"].to_f) > 0 + $stderr.puts "request started pid=#{Process.pid}" + sleep seconds + end + [200, {}, ["done\n"]] + } + RUBY + + test "TTIN during a graceful shutdown doesn't spawn workers" do + addr, port = unused_port + pid = spawn_server(app: write_app, config: <<~RUBY) + listen "#{addr}:#{port}" + worker_processes 1 + RUBY + start_slow_request(addr, port, 3) + + Process.kill(:QUIT, pid) + assert_stderr("QUIT received, starting graceful shutdown") + Process.kill(:TTIN, pid) + + assert_exited(pid, 0, timeout: 6) + refute_stderr(/worker=1|No mold alive/) + end + + private + + def write_app + File.write("slow.ru", SLOW_APP) + File.join(@pwd, "slow.ru") + end + + def start_slow_request(addr, port, seconds) + assert_healthy("http://#{addr}:#{port}") + Thread.new { http_get("http://#{addr}:#{port}/?#{seconds}", timeout: seconds + 5) rescue nil } + assert_stderr(/request started pid=\d+/, timeout: 5) + Integer(read_stderr[/request started pid=(\d+)/, 1]) + end +end From c45abb7360161f4998f3d2ee6073e189a3fe9342 Mon Sep 17 00:00:00 2001 From: Brian Scanlan Date: Thu, 19 Nov 2026 13:15:40 +0000 Subject: [PATCH 2/6] Stop busy-looping the monitor during shutdown Commit 0cd51fcd replaced `sleep(0.1)` in the shutdown loop of `HttpServer#stop` with `monitor_loop(false)`, which does not sleep. While workers drain, the monitor spins on one CPU core for up to `timeout` seconds. Because the loop never calls `monitor_sleep`, the monitor also never reads messages such as `WorkerSpawned` from the control socket. Wait on the control socket for at most 0.1 seconds per iteration. A child exit, a signal, or a message wakes the monitor immediately. --- CHANGELOG.md | 1 + lib/pitchfork/http_server.rb | 1 + test/integration/test_shutdown.rb | 20 ++++++++++++++++++++ 3 files changed, 22 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 61daa3ba..73bfc989 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ # Unreleased +- Fix the monitor using a full CPU core while it waits for workers during a graceful shutdown. - Fix `TTIN` during a graceful shutdown spawning new workers. - Preserve worker spawn deadlines when the mold starts workers. diff --git a/lib/pitchfork/http_server.rb b/lib/pitchfork/http_server.rb index c9bcf575..ff6dbada 100644 --- a/lib/pitchfork/http_server.rb +++ b/lib/pitchfork/http_server.rb @@ -471,6 +471,7 @@ def stop(graceful = true) else @children.hard_kill_all(:INT) end + monitor_sleep(0.1) if monitor_loop(false) == StopIteration return StopIteration end diff --git a/test/integration/test_shutdown.rb b/test/integration/test_shutdown.rb index 19947660..77159ff4 100644 --- a/test/integration/test_shutdown.rb +++ b/test/integration/test_shutdown.rb @@ -28,6 +28,22 @@ class TestShutdown < Pitchfork::IntegrationTest refute_stderr(/worker=1|No mold alive/) end + test "graceful shutdown doesn't busy-loop the monitor" do + addr, port = unused_port + pid = spawn_server(app: write_app, config: <<~RUBY) + listen "#{addr}:#{port}" + worker_processes 1 + RUBY + start_slow_request(addr, port, 4) + + cpu_before = cpu_time(pid) + Process.kill(:QUIT, pid) + sleep 3 + assert_operator cpu_time(pid) - cpu_before, :<, 1.5 + + assert_exited(pid, 0, timeout: 4) + end + private def write_app @@ -41,4 +57,8 @@ def start_slow_request(addr, port, seconds) assert_stderr(/request started pid=\d+/, timeout: 5) Integer(read_stderr[/request started pid=(\d+)/, 1]) end + + def cpu_time(pid) + `ps -o time= -p #{pid}`.split(":").map(&:to_f).reverse.each_with_index.sum { |n, i| n * 60**i } + end end From ec266788888ffda2df9354d6c7a3f9280e18d2df Mon Sep 17 00:00:00 2001 From: Brian Scanlan Date: Thu, 19 Nov 2026 13:16:19 +0000 Subject: [PATCH 3/6] Keep waiting for workers after a repeated QUIT or TERM `HttpServer#stop` returned early when `monitor_loop` handled a signal that returns `StopIteration`. For `INT`, the nested `stop(false)` has already killed the workers. For `QUIT` and `TERM`, nothing has been done yet. A second `QUIT` or `TERM` during a graceful shutdown made the monitor exit while workers still handled requests, and the monitor no longer enforced the hard timeout on them. Ignore the return value. After a nested `stop(false)`, no children remain, so the loop ends immediately. A new test makes sure that `INT` still ends a graceful shutdown immediately. --- CHANGELOG.md | 1 + lib/pitchfork/http_server.rb | 4 +--- test/integration/test_shutdown.rb | 37 +++++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 73bfc989..9d3e7227 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Unreleased - Fix the monitor using a full CPU core while it waits for workers during a graceful shutdown. +- Fix a second `QUIT` or `TERM` during a graceful shutdown making the monitor exit before its workers. - Fix `TTIN` during a graceful shutdown spawning new workers. - Preserve worker spawn deadlines when the mold starts workers. diff --git a/lib/pitchfork/http_server.rb b/lib/pitchfork/http_server.rb index ff6dbada..f9b19eb3 100644 --- a/lib/pitchfork/http_server.rb +++ b/lib/pitchfork/http_server.rb @@ -472,9 +472,7 @@ def stop(graceful = true) @children.hard_kill_all(:INT) end monitor_sleep(0.1) - if monitor_loop(false) == StopIteration - return StopIteration - end + monitor_loop(false) end @children.each do |child| diff --git a/test/integration/test_shutdown.rb b/test/integration/test_shutdown.rb index 77159ff4..a2a83a90 100644 --- a/test/integration/test_shutdown.rb +++ b/test/integration/test_shutdown.rb @@ -44,6 +44,43 @@ class TestShutdown < Pitchfork::IntegrationTest assert_exited(pid, 0, timeout: 4) end + test "repeated QUIT still waits for workers to exit" do + addr, port = unused_port + pid = spawn_server(app: write_app, config: <<~RUBY) + listen "#{addr}:#{port}" + worker_processes 1 + timeout 3 + RUBY + worker_pid = start_slow_request(addr, port, 30) + + Process.kill(:QUIT, pid) + assert_stderr("QUIT received, starting graceful shutdown") + Process.kill(:QUIT, pid) + + assert_exited(pid, 0, timeout: 8) + assert_raises(Errno::ESRCH) { Process.kill(0, worker_pid) } + ensure + Process.kill(:KILL, worker_pid) rescue nil + end + + test "INT during a graceful shutdown kills workers immediately" do + addr, port = unused_port + pid = spawn_server(app: write_app, config: <<~RUBY) + listen "#{addr}:#{port}" + worker_processes 1 + RUBY + worker_pid = start_slow_request(addr, port, 30) + + Process.kill(:QUIT, pid) + assert_stderr("QUIT received, starting graceful shutdown") + Process.kill(:INT, pid) + + assert_exited(pid, 0, timeout: 2) + assert_raises(Errno::ESRCH) { Process.kill(0, worker_pid) } + ensure + Process.kill(:KILL, worker_pid) rescue nil + end + private def write_app From e1219dcd780ca10bcd2badefec16bc31d541de48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20Mendon=C3=A7a=20Fran=C3=A7a?= Date: Thu, 19 Nov 2026 11:29:21 -0500 Subject: [PATCH 4/6] Wait for pending workers during shutdown --- CHANGELOG.md | 2 +- lib/pitchfork/http_server.rb | 8 ++- test/integration/test_shutdown.rb | 106 ++++++++++++++++++++++++++++++ 3 files changed, 112 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d3e7227..53ce14a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # Unreleased - Fix the monitor using a full CPU core while it waits for workers during a graceful shutdown. -- Fix a second `QUIT` or `TERM` during a graceful shutdown making the monitor exit before its workers. +- Fix a second `QUIT` or `TERM` during a graceful shutdown making the monitor exit before pending or running workers. - Fix `TTIN` during a graceful shutdown spawning new workers. - Preserve worker spawn deadlines when the mold starts workers. diff --git a/lib/pitchfork/http_server.rb b/lib/pitchfork/http_server.rb index f9b19eb3..bff88a7a 100644 --- a/lib/pitchfork/http_server.rb +++ b/lib/pitchfork/http_server.rb @@ -461,7 +461,7 @@ def stop(graceful = true) proc_name role: 'monitor', status: 'shutting down' @respawn = false SharedMemory.shutting_down! - wait_for_pending_workers + wait_for_pending_workers(stop_on_signal: false) Pitchfork.close_listeners limit = Pitchfork.time_now + timeout @@ -874,10 +874,10 @@ def spawn_missing_workers exit! end - def wait_for_pending_workers + def wait_for_pending_workers(stop_on_signal: true) while @children.pending_workers? monitor_sleep(0.5) - if monitor_loop(false) == StopIteration + if monitor_loop(false) == StopIteration && stop_on_signal return StopIteration end end @@ -1079,6 +1079,7 @@ def init_worker_process(worker) proc_name role: "(#{worker.version}.#{worker.generation}) worker[#{worker.nr}]", status: "init" worker.reset worker.register_to_monitor(@control_socket[1]) + exit!(0) if SharedMemory.shutting_down? # we'll re-trap :QUIT and :TERM later for graceful shutdown iff we accept clients exit_sigs = [ :QUIT, :TERM, :INT ] exit_sigs.each { |sig| trap(sig) { exit!(0) } } @@ -1105,6 +1106,7 @@ def init_service_process(service) proc_name role: "(#{service.version}.#{service.generation}) service", status: "init" Pitchfork.close_listeners # Don't appear as listening to incoming requests service.register_to_monitor(@control_socket[1]) + exit!(0) if SharedMemory.shutting_down? readers = [service] trap(:QUIT) { nuke_listeners!(readers) } trap(:TERM) { nuke_listeners!(readers) } diff --git a/test/integration/test_shutdown.rb b/test/integration/test_shutdown.rb index a2a83a90..8c9d6bb6 100644 --- a/test/integration/test_shutdown.rb +++ b/test/integration/test_shutdown.rb @@ -63,6 +63,81 @@ class TestShutdown < Pitchfork::IntegrationTest Process.kill(:KILL, worker_pid) rescue nil end + test "repeated QUIT waits for a pending worker" do + addr, port = unused_port + pid = spawn_pending_worker(addr, port, spawn_timeout: 10) + assert_healthy("http://#{addr}:#{port}") + Process.kill(:TTIN, pid) + assert_stderr(/pending worker pid=\d+/, timeout: 5) + worker_pid = Integer(read_stderr[/pending worker pid=(\d+)/, 1]) + + Process.kill(:QUIT, pid) + assert_stderr("QUIT received, starting graceful shutdown") + Process.kill(:QUIT, pid) + assert_stderr(/QUIT received, starting graceful shutdown.*QUIT received, starting graceful shutdown/m, timeout: 5) + sleep 0.2 + assert_nil Process.waitpid(pid, Process::WNOHANG) + + File.write("continue-worker", "") + assert_exited(pid, 0, timeout: 5) + assert_raises(Errno::ESRCH) { Process.kill(0, worker_pid) } + ensure + Process.kill(:KILL, worker_pid) rescue nil + end + + test "a worker that exceeds its spawn timeout exits after shutdown" do + addr, port = unused_port + pid = spawn_pending_worker(addr, port, spawn_timeout: 1) + assert_healthy("http://#{addr}:#{port}") + Process.kill(:TTIN, pid) + assert_stderr(/pending worker pid=\d+/, timeout: 5) + worker_pid = Integer(read_stderr[/pending worker pid=(\d+)/, 1]) + + Process.kill(:QUIT, pid) + assert_stderr("QUIT received, starting graceful shutdown") + Process.kill(:QUIT, pid) + assert_exited(pid, 0, timeout: 6) + + File.write("continue-worker", "") + assert_stderr("pending worker continuing", timeout: 5) + assert_process_exits(worker_pid) + worker_pid = nil + ensure + Process.kill(:KILL, worker_pid) rescue nil + end + + test "a pending service worker exits after shutdown" do + addr, port = unused_port + pid = spawn_server(app: write_app, config: <<~RUBY) + listen "#{addr}:#{port}" + worker_processes 1 + + module SlowPendingService + def init_service_process(service) + $stderr.puts "pending service pid=\#{Process.pid}" + sleep 0.01 until File.exist?("continue-service") + $stderr.puts "pending service continuing" + super + end + end + + after_mold_fork { |server, _mold| server.extend(SlowPendingService) } + before_service_worker_ready { |_server, _service| } + RUBY + assert_healthy("http://#{addr}:#{port}") + assert_stderr(/pending service pid=\d+/, timeout: 5) + service_pid = Integer(read_stderr[/pending service pid=(\d+)/, 1]) + + Process.kill(:QUIT, pid) + assert_exited(pid, 0, timeout: 5) + File.write("continue-service", "") + assert_stderr("pending service continuing", timeout: 5) + assert_process_exits(service_pid) + service_pid = nil + ensure + Process.kill(:KILL, service_pid) rescue nil + end + test "INT during a graceful shutdown kills workers immediately" do addr, port = unused_port pid = spawn_server(app: write_app, config: <<~RUBY) @@ -83,6 +158,37 @@ class TestShutdown < Pitchfork::IntegrationTest private + def assert_process_exits(pid) + 50.times do + Process.kill(0, pid) + sleep 0.1 + rescue Errno::ESRCH + return + end + flunk "process pid=#{pid} did not exit" + end + + def spawn_pending_worker(addr, port, spawn_timeout:) + spawn_server(app: write_app, config: <<~RUBY) + listen "#{addr}:#{port}" + worker_processes 1 + spawn_timeout #{spawn_timeout} + + module SlowPendingWorker + def init_worker_process(worker) + if worker.nr == 1 + $stderr.puts "pending worker pid=\#{Process.pid}" + sleep 0.01 until File.exist?("continue-worker") + $stderr.puts "pending worker continuing" + end + super + end + end + + after_mold_fork { |server, _mold| server.extend(SlowPendingWorker) } + RUBY + end + def write_app File.write("slow.ru", SLOW_APP) File.join(@pwd, "slow.ru") From 74fae88da6e4ed302152193c247403be2df0aae5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20Mendon=C3=A7a=20Fran=C3=A7a?= Date: Thu, 19 Nov 2026 11:30:44 -0500 Subject: [PATCH 5/6] Prevent restarts during shutdown --- CHANGELOG.md | 1 + lib/pitchfork/http_server.rb | 6 +++++- test/integration/test_shutdown.rb | 18 ++++++++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53ce14a2..2782451f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ - Fix the monitor using a full CPU core while it waits for workers during a graceful shutdown. - Fix a second `QUIT` or `TERM` during a graceful shutdown making the monitor exit before pending or running workers. - Fix `TTIN` during a graceful shutdown spawning new workers. +- Prevent `USR1` from restarting the monitor during a graceful shutdown. - Preserve worker spawn deadlines when the mold starts workers. # 0.18.4 diff --git a/lib/pitchfork/http_server.rb b/lib/pitchfork/http_server.rb index bff88a7a..08ac3166 100644 --- a/lib/pitchfork/http_server.rb +++ b/lib/pitchfork/http_server.rb @@ -403,7 +403,11 @@ def monitor_handle_message(message) stop(false) return StopIteration when :USR1 # trigger a hot restart - trigger_restart + if SharedMemory.shutting_down? + logger.error "Can't trigger a restart as the server is shutting down" + else + trigger_restart + end when :USR2 # trigger a promotion if @respawn trigger_refork diff --git a/test/integration/test_shutdown.rb b/test/integration/test_shutdown.rb index 8c9d6bb6..90c99878 100644 --- a/test/integration/test_shutdown.rb +++ b/test/integration/test_shutdown.rb @@ -138,6 +138,24 @@ def init_service_process(service) Process.kill(:KILL, service_pid) rescue nil end + test "USR1 during a graceful shutdown does not restart the monitor" do + addr, port = unused_port + pid = spawn_server(app: write_app, config: <<~RUBY) + listen "#{addr}:#{port}" + worker_processes 1 + RUBY + start_slow_request(addr, port, 3) + + Process.kill(:QUIT, pid) + assert_stderr("QUIT received, starting graceful shutdown") + Process.kill(:QUIT, pid) + assert_stderr(/QUIT received, starting graceful shutdown.*QUIT received, starting graceful shutdown/m, timeout: 5) + Process.kill(:USR1, pid) + + assert_stderr("Can't trigger a restart as the server is shutting down", timeout: 3) + assert_exited(pid, 0, timeout: 6) + end + test "INT during a graceful shutdown kills workers immediately" do addr, port = unused_port pid = spawn_server(app: write_app, config: <<~RUBY) From 7babda19542b5007c6474482332567b12242db8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20Mendon=C3=A7a=20Fran=C3=A7a?= Date: Thu, 19 Nov 2026 13:04:02 -0500 Subject: [PATCH 6/6] Consolidate shutdown control --- CHANGELOG.md | 1 + lib/pitchfork/http_server.rb | 47 +++++++++++--------- test/integration/test_shutdown.rb | 71 +++++++++++++++++++++++++++++++ 3 files changed, 98 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2782451f..fe8a8af6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - Fix a second `QUIT` or `TERM` during a graceful shutdown making the monitor exit before pending or running workers. - Fix `TTIN` during a graceful shutdown spawning new workers. - Prevent `USR1` from restarting the monitor during a graceful shutdown. +- Honor shutdown signals while the server is starting. - Preserve worker spawn deadlines when the mold starts workers. # 0.18.4 diff --git a/lib/pitchfork/http_server.rb b/lib/pitchfork/http_server.rb index 08ac3166..9c5d8bc0 100644 --- a/lib/pitchfork/http_server.rb +++ b/lib/pitchfork/http_server.rb @@ -112,6 +112,7 @@ def initialize(app, options = {}) @exit_status = 0 @app = app @respawn = false + @graceful_shutdown = true @refork_max_unavailable = nil @consecutive_spawn_errors = 0 @max_consecutive_spawn_errors = nil @@ -226,7 +227,7 @@ def start(sync = true) if REFORKING_AVAILABLE spawn_initial_mold - wait_for_pending_workers + return self if wait_for_pending_workers == StopIteration unless @children.mold raise BootFailure, "The initial mold failed to boot" end @@ -241,7 +242,7 @@ def start(sync = true) # We could just return here as we'd register them later in #join. # However a good part of the test suite assumes #start only return # once all initial workers are spawned. - wait_for_pending_workers + return self if wait_for_pending_workers == StopIteration end @after_monitor_ready&.call(self) @@ -324,7 +325,7 @@ def listen(address, opt = listener_opts[address] || {}) # one-at-a-time time and we'll happily drop signals in case somebody # is signalling us too often. def join - @respawn = true + @respawn = !SharedMemory.shutting_down? proc_name role: 'monitor', status: ARGV.join(' ') @@ -336,7 +337,7 @@ def join end @ready_pipe = @ready_pipe.close rescue nil end - while true + until SharedMemory.shutting_down? begin if monitor_loop == StopIteration break @@ -345,7 +346,7 @@ def join Pitchfork.log_error(@logger, "monitor loop error", e) end end - stop # gracefully shutdown all workers on our way out + stop logger.info "monitor complete status=#{@exit_status}" @exit_status end @@ -376,8 +377,7 @@ def monitor_loop(sleep = true) if @max_consecutive_spawn_errors && @consecutive_spawn_errors > @max_consecutive_spawn_errors && !SharedMemory.shutting_down? logger.fatal("#{@consecutive_spawn_errors} consecutive failures to spawn children, aborting - broken after_worker_fork callback?") @exit_status = 1 - SharedMemory.shutting_down! - stop(false) + request_shutdown(false) return StopIteration end else @@ -393,14 +393,10 @@ def monitor_loop(sleep = true) def monitor_handle_message(message) case message - when :QUIT, :TERM # graceful shutdown - SharedMemory.shutting_down! - logger.info "#{message} received, starting graceful shutdown" - return StopIteration - when :INT # immediate shutdown - SharedMemory.shutting_down! - logger.info "#{message} received, starting immediate shutdown" - stop(false) + when :QUIT, :TERM, :INT + graceful = message != :INT + request_shutdown(graceful) + logger.info "#{message} received, starting #{graceful ? 'graceful' : 'immediate'} shutdown" return StopIteration when :USR1 # trigger a hot restart if SharedMemory.shutting_down? @@ -464,13 +460,13 @@ def monitor_handle_message(message) def stop(graceful = true) proc_name role: 'monitor', status: 'shutting down' @respawn = false - SharedMemory.shutting_down! + request_shutdown(graceful) wait_for_pending_workers(stop_on_signal: false) Pitchfork.close_listeners limit = Pitchfork.time_now + timeout until @children.empty? || Pitchfork.time_now > limit - if graceful + if @graceful_shutdown @children.soft_kill_all(:TERM) else @children.hard_kill_all(:INT) @@ -530,6 +526,11 @@ def check_client_connection=(bool) private + def request_shutdown(graceful) + @graceful_shutdown = false unless graceful + SharedMemory.shutting_down! + end + def refork_max_unavailable @refork_max_unavailable ||= (worker_processes * 0.1).ceil end @@ -1075,6 +1076,12 @@ def reset_signal_handlers [:QUIT, :TERM, :INT].each { |sig| trap(sig) { exit!(0) } } end + def register_child_and_exit_if_shutting_down(child) + # Register first so the monitor can account for the child during shutdown. + child.register_to_monitor(@control_socket[1]) + exit!(0) if SharedMemory.shutting_down? + end + # gets rid of stuff the worker has no business keeping track of # to free some resources and drops all sig handlers. # traps for USR2, and HUP may be set in the after_worker_fork/after_mold_fork Procs @@ -1082,8 +1089,7 @@ def reset_signal_handlers def init_worker_process(worker) proc_name role: "(#{worker.version}.#{worker.generation}) worker[#{worker.nr}]", status: "init" worker.reset - worker.register_to_monitor(@control_socket[1]) - exit!(0) if SharedMemory.shutting_down? + register_child_and_exit_if_shutting_down(worker) # we'll re-trap :QUIT and :TERM later for graceful shutdown iff we accept clients exit_sigs = [ :QUIT, :TERM, :INT ] exit_sigs.each { |sig| trap(sig) { exit!(0) } } @@ -1109,8 +1115,7 @@ def init_worker_process(worker) def init_service_process(service) proc_name role: "(#{service.version}.#{service.generation}) service", status: "init" Pitchfork.close_listeners # Don't appear as listening to incoming requests - service.register_to_monitor(@control_socket[1]) - exit!(0) if SharedMemory.shutting_down? + register_child_and_exit_if_shutting_down(service) readers = [service] trap(:QUIT) { nuke_listeners!(readers) } trap(:TERM) { nuke_listeners!(readers) } diff --git a/test/integration/test_shutdown.rb b/test/integration/test_shutdown.rb index 90c99878..96e0bea2 100644 --- a/test/integration/test_shutdown.rb +++ b/test/integration/test_shutdown.rb @@ -156,6 +156,40 @@ def init_service_process(service) assert_exited(pid, 0, timeout: 6) end + test "QUIT during startup shuts down the server" do + assert_shutdown_during_startup(:QUIT) + end + + test "INT during startup shuts down the server" do + assert_shutdown_during_startup(:INT) + end + + test "INT during shutdown stays immediate while a worker is pending" do + addr, port = unused_port + pid = spawn_pending_worker(addr, port, spawn_timeout: 10) + active_worker_pid = start_slow_request(addr, port, 30) + Process.kill(:TTIN, pid) + assert_stderr(/pending worker pid=\d+/, timeout: 5) + pending_worker_pid = Integer(read_stderr[/pending worker pid=(\d+)/, 1]) + + Process.kill(:QUIT, pid) + assert_stderr("QUIT received, starting graceful shutdown") + Process.kill(:INT, pid) + assert_stderr("INT received, starting immediate shutdown") + Process.kill(:QUIT, pid) + assert_stderr(/QUIT received, starting graceful shutdown.*QUIT received, starting graceful shutdown/m, timeout: 5) + File.write("continue-worker", "") + + assert_exited(pid, 0, timeout: 4) + assert_process_exits(active_worker_pid) + active_worker_pid = nil + assert_process_exits(pending_worker_pid) + pending_worker_pid = nil + ensure + Process.kill(:KILL, active_worker_pid) rescue nil + Process.kill(:KILL, pending_worker_pid) rescue nil + end + test "INT during a graceful shutdown kills workers immediately" do addr, port = unused_port pid = spawn_server(app: write_app, config: <<~RUBY) @@ -176,6 +210,43 @@ def init_service_process(service) private + def assert_shutdown_during_startup(signal) + addr, port = unused_port + pid = spawn_server(app: write_app, config: <<~RUBY) + listen "#{addr}:#{port}" + worker_processes 1 + + module SynchronousStartup + def start(_sync = true) + super(true) + end + end + Pitchfork::HttpServer.prepend(SynchronousStartup) + + module SlowInitialWorker + def init_worker_process(worker) + $stderr.puts "initial worker pid=\#{Process.pid}" + sleep 0.01 until File.exist?("continue-initial-worker") + super + end + end + after_mold_fork { |server, _mold| server.extend(SlowInitialWorker) } + RUBY + assert_stderr(/initial worker pid=\d+/, timeout: 5) + worker_pid = Integer(read_stderr[/initial worker pid=(\d+)/, 1]) + + Process.kill(signal, pid) + mode = signal == :INT ? "immediate" : "graceful" + assert_stderr("#{signal} received, starting #{mode} shutdown", timeout: 5) + File.write("continue-initial-worker", "") + + assert_exited(pid, 0, timeout: 5) + assert_process_exits(worker_pid) + worker_pid = nil + ensure + Process.kill(:KILL, worker_pid) rescue nil + end + def assert_process_exits(pid) 50.times do Process.kill(0, pid)