diff --git a/CHANGELOG.md b/CHANGELOG.md index ec3f9ef0..fe8a8af6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Unreleased +- Fix the monitor using a full CPU core while it waits for workers during a graceful shutdown. +- Fix a second `QUIT` or `TERM` during a graceful shutdown making the monitor exit before pending or running workers. +- Fix `TTIN` during a graceful shutdown spawning new workers. +- Prevent `USR1` from restarting the monitor during a graceful shutdown. +- Honor shutdown signals while the server is starting. - Preserve worker spawn deadlines when the mold starts workers. # 0.18.4 diff --git a/lib/pitchfork/http_server.rb b/lib/pitchfork/http_server.rb index 5056e098..9c5d8bc0 100644 --- a/lib/pitchfork/http_server.rb +++ b/lib/pitchfork/http_server.rb @@ -112,6 +112,7 @@ def initialize(app, options = {}) @exit_status = 0 @app = app @respawn = false + @graceful_shutdown = true @refork_max_unavailable = nil @consecutive_spawn_errors = 0 @max_consecutive_spawn_errors = nil @@ -226,7 +227,7 @@ def start(sync = true) if REFORKING_AVAILABLE spawn_initial_mold - wait_for_pending_workers + return self if wait_for_pending_workers == StopIteration unless @children.mold raise BootFailure, "The initial mold failed to boot" end @@ -241,7 +242,7 @@ def start(sync = true) # We could just return here as we'd register them later in #join. # However a good part of the test suite assumes #start only return # once all initial workers are spawned. - wait_for_pending_workers + return self if wait_for_pending_workers == StopIteration end @after_monitor_ready&.call(self) @@ -324,7 +325,7 @@ def listen(address, opt = listener_opts[address] || {}) # one-at-a-time time and we'll happily drop signals in case somebody # is signalling us too often. def join - @respawn = true + @respawn = !SharedMemory.shutting_down? proc_name role: 'monitor', status: ARGV.join(' ') @@ -336,7 +337,7 @@ def join end @ready_pipe = @ready_pipe.close rescue nil end - while true + until SharedMemory.shutting_down? begin if monitor_loop == StopIteration break @@ -345,7 +346,7 @@ def join Pitchfork.log_error(@logger, "monitor loop error", e) end end - stop # gracefully shutdown all workers on our way out + stop logger.info "monitor complete status=#{@exit_status}" @exit_status end @@ -376,8 +377,7 @@ def monitor_loop(sleep = true) if @max_consecutive_spawn_errors && @consecutive_spawn_errors > @max_consecutive_spawn_errors && !SharedMemory.shutting_down? logger.fatal("#{@consecutive_spawn_errors} consecutive failures to spawn children, aborting - broken after_worker_fork callback?") @exit_status = 1 - SharedMemory.shutting_down! - stop(false) + request_shutdown(false) return StopIteration end else @@ -393,17 +393,17 @@ def monitor_loop(sleep = true) def monitor_handle_message(message) case message - when :QUIT, :TERM # graceful shutdown - SharedMemory.shutting_down! - logger.info "#{message} received, starting graceful shutdown" - return StopIteration - when :INT # immediate shutdown - SharedMemory.shutting_down! - logger.info "#{message} received, starting immediate shutdown" - stop(false) + when :QUIT, :TERM, :INT + graceful = message != :INT + request_shutdown(graceful) + logger.info "#{message} received, starting #{graceful ? 'graceful' : 'immediate'} shutdown" return StopIteration when :USR1 # trigger a hot restart - trigger_restart + if SharedMemory.shutting_down? + logger.error "Can't trigger a restart as the server is shutting down" + else + trigger_restart + end when :USR2 # trigger a promotion if @respawn trigger_refork @@ -411,7 +411,6 @@ def monitor_handle_message(message) logger.error "Can't trigger a refork as the server is shutting down" end when :TTIN - @respawn = true self.worker_processes += 1 when :TTOU self.worker_processes -= 1 if self.worker_processes > 0 @@ -461,20 +460,19 @@ def monitor_handle_message(message) def stop(graceful = true) proc_name role: 'monitor', status: 'shutting down' @respawn = false - SharedMemory.shutting_down! - wait_for_pending_workers + request_shutdown(graceful) + wait_for_pending_workers(stop_on_signal: false) Pitchfork.close_listeners limit = Pitchfork.time_now + timeout until @children.empty? || Pitchfork.time_now > limit - if graceful + if @graceful_shutdown @children.soft_kill_all(:TERM) else @children.hard_kill_all(:INT) end - if monitor_loop(false) == StopIteration - return StopIteration - end + monitor_sleep(0.1) + monitor_loop(false) end @children.each do |child| @@ -528,6 +526,11 @@ def check_client_connection=(bool) private + def request_shutdown(graceful) + @graceful_shutdown = false unless graceful + SharedMemory.shutting_down! + end + def refork_max_unavailable @refork_max_unavailable ||= (worker_processes * 0.1).ceil end @@ -876,10 +879,10 @@ def spawn_missing_workers exit! end - def wait_for_pending_workers + def wait_for_pending_workers(stop_on_signal: true) while @children.pending_workers? monitor_sleep(0.5) - if monitor_loop(false) == StopIteration + if monitor_loop(false) == StopIteration && stop_on_signal return StopIteration end end @@ -1073,6 +1076,12 @@ def reset_signal_handlers [:QUIT, :TERM, :INT].each { |sig| trap(sig) { exit!(0) } } end + def register_child_and_exit_if_shutting_down(child) + # Register first so the monitor can account for the child during shutdown. + child.register_to_monitor(@control_socket[1]) + exit!(0) if SharedMemory.shutting_down? + end + # gets rid of stuff the worker has no business keeping track of # to free some resources and drops all sig handlers. # traps for USR2, and HUP may be set in the after_worker_fork/after_mold_fork Procs @@ -1080,7 +1089,7 @@ def reset_signal_handlers def init_worker_process(worker) proc_name role: "(#{worker.version}.#{worker.generation}) worker[#{worker.nr}]", status: "init" worker.reset - worker.register_to_monitor(@control_socket[1]) + register_child_and_exit_if_shutting_down(worker) # we'll re-trap :QUIT and :TERM later for graceful shutdown iff we accept clients exit_sigs = [ :QUIT, :TERM, :INT ] exit_sigs.each { |sig| trap(sig) { exit!(0) } } @@ -1106,7 +1115,7 @@ def init_worker_process(worker) def init_service_process(service) proc_name role: "(#{service.version}.#{service.generation}) service", status: "init" Pitchfork.close_listeners # Don't appear as listening to incoming requests - service.register_to_monitor(@control_socket[1]) + register_child_and_exit_if_shutting_down(service) readers = [service] trap(:QUIT) { nuke_listeners!(readers) } trap(:TERM) { nuke_listeners!(readers) } diff --git a/test/integration/test_shutdown.rb b/test/integration/test_shutdown.rb new file mode 100644 index 00000000..96e0bea2 --- /dev/null +++ b/test/integration/test_shutdown.rb @@ -0,0 +1,296 @@ +# frozen_string_literal: true +require 'integration_test_helper' + +class TestShutdown < Pitchfork::IntegrationTest + SLOW_APP = <<~'RUBY' + run lambda { |env| + if (seconds = env["QUERY_STRING"].to_f) > 0 + $stderr.puts "request started pid=#{Process.pid}" + sleep seconds + end + [200, {}, ["done\n"]] + } + RUBY + + test "TTIN during a graceful shutdown doesn't spawn workers" do + addr, port = unused_port + pid = spawn_server(app: write_app, config: <<~RUBY) + listen "#{addr}:#{port}" + worker_processes 1 + RUBY + start_slow_request(addr, port, 3) + + Process.kill(:QUIT, pid) + assert_stderr("QUIT received, starting graceful shutdown") + Process.kill(:TTIN, pid) + + assert_exited(pid, 0, timeout: 6) + refute_stderr(/worker=1|No mold alive/) + end + + test "graceful shutdown doesn't busy-loop the monitor" do + addr, port = unused_port + pid = spawn_server(app: write_app, config: <<~RUBY) + listen "#{addr}:#{port}" + worker_processes 1 + RUBY + start_slow_request(addr, port, 4) + + cpu_before = cpu_time(pid) + Process.kill(:QUIT, pid) + sleep 3 + assert_operator cpu_time(pid) - cpu_before, :<, 1.5 + + assert_exited(pid, 0, timeout: 4) + end + + test "repeated QUIT still waits for workers to exit" do + addr, port = unused_port + pid = spawn_server(app: write_app, config: <<~RUBY) + listen "#{addr}:#{port}" + worker_processes 1 + timeout 3 + RUBY + worker_pid = start_slow_request(addr, port, 30) + + Process.kill(:QUIT, pid) + assert_stderr("QUIT received, starting graceful shutdown") + Process.kill(:QUIT, pid) + + assert_exited(pid, 0, timeout: 8) + assert_raises(Errno::ESRCH) { Process.kill(0, worker_pid) } + ensure + Process.kill(:KILL, worker_pid) rescue nil + end + + test "repeated QUIT waits for a pending worker" do + addr, port = unused_port + pid = spawn_pending_worker(addr, port, spawn_timeout: 10) + assert_healthy("http://#{addr}:#{port}") + Process.kill(:TTIN, pid) + assert_stderr(/pending worker pid=\d+/, timeout: 5) + worker_pid = Integer(read_stderr[/pending worker pid=(\d+)/, 1]) + + Process.kill(:QUIT, pid) + assert_stderr("QUIT received, starting graceful shutdown") + Process.kill(:QUIT, pid) + assert_stderr(/QUIT received, starting graceful shutdown.*QUIT received, starting graceful shutdown/m, timeout: 5) + sleep 0.2 + assert_nil Process.waitpid(pid, Process::WNOHANG) + + File.write("continue-worker", "") + assert_exited(pid, 0, timeout: 5) + assert_raises(Errno::ESRCH) { Process.kill(0, worker_pid) } + ensure + Process.kill(:KILL, worker_pid) rescue nil + end + + test "a worker that exceeds its spawn timeout exits after shutdown" do + addr, port = unused_port + pid = spawn_pending_worker(addr, port, spawn_timeout: 1) + assert_healthy("http://#{addr}:#{port}") + Process.kill(:TTIN, pid) + assert_stderr(/pending worker pid=\d+/, timeout: 5) + worker_pid = Integer(read_stderr[/pending worker pid=(\d+)/, 1]) + + Process.kill(:QUIT, pid) + assert_stderr("QUIT received, starting graceful shutdown") + Process.kill(:QUIT, pid) + assert_exited(pid, 0, timeout: 6) + + File.write("continue-worker", "") + assert_stderr("pending worker continuing", timeout: 5) + assert_process_exits(worker_pid) + worker_pid = nil + ensure + Process.kill(:KILL, worker_pid) rescue nil + end + + test "a pending service worker exits after shutdown" do + addr, port = unused_port + pid = spawn_server(app: write_app, config: <<~RUBY) + listen "#{addr}:#{port}" + worker_processes 1 + + module SlowPendingService + def init_service_process(service) + $stderr.puts "pending service pid=\#{Process.pid}" + sleep 0.01 until File.exist?("continue-service") + $stderr.puts "pending service continuing" + super + end + end + + after_mold_fork { |server, _mold| server.extend(SlowPendingService) } + before_service_worker_ready { |_server, _service| } + RUBY + assert_healthy("http://#{addr}:#{port}") + assert_stderr(/pending service pid=\d+/, timeout: 5) + service_pid = Integer(read_stderr[/pending service pid=(\d+)/, 1]) + + Process.kill(:QUIT, pid) + assert_exited(pid, 0, timeout: 5) + File.write("continue-service", "") + assert_stderr("pending service continuing", timeout: 5) + assert_process_exits(service_pid) + service_pid = nil + ensure + Process.kill(:KILL, service_pid) rescue nil + end + + test "USR1 during a graceful shutdown does not restart the monitor" do + addr, port = unused_port + pid = spawn_server(app: write_app, config: <<~RUBY) + listen "#{addr}:#{port}" + worker_processes 1 + RUBY + start_slow_request(addr, port, 3) + + Process.kill(:QUIT, pid) + assert_stderr("QUIT received, starting graceful shutdown") + Process.kill(:QUIT, pid) + assert_stderr(/QUIT received, starting graceful shutdown.*QUIT received, starting graceful shutdown/m, timeout: 5) + Process.kill(:USR1, pid) + + assert_stderr("Can't trigger a restart as the server is shutting down", timeout: 3) + assert_exited(pid, 0, timeout: 6) + end + + test "QUIT during startup shuts down the server" do + assert_shutdown_during_startup(:QUIT) + end + + test "INT during startup shuts down the server" do + assert_shutdown_during_startup(:INT) + end + + test "INT during shutdown stays immediate while a worker is pending" do + addr, port = unused_port + pid = spawn_pending_worker(addr, port, spawn_timeout: 10) + active_worker_pid = start_slow_request(addr, port, 30) + Process.kill(:TTIN, pid) + assert_stderr(/pending worker pid=\d+/, timeout: 5) + pending_worker_pid = Integer(read_stderr[/pending worker pid=(\d+)/, 1]) + + Process.kill(:QUIT, pid) + assert_stderr("QUIT received, starting graceful shutdown") + Process.kill(:INT, pid) + assert_stderr("INT received, starting immediate shutdown") + Process.kill(:QUIT, pid) + assert_stderr(/QUIT received, starting graceful shutdown.*QUIT received, starting graceful shutdown/m, timeout: 5) + File.write("continue-worker", "") + + assert_exited(pid, 0, timeout: 4) + assert_process_exits(active_worker_pid) + active_worker_pid = nil + assert_process_exits(pending_worker_pid) + pending_worker_pid = nil + ensure + Process.kill(:KILL, active_worker_pid) rescue nil + Process.kill(:KILL, pending_worker_pid) rescue nil + end + + test "INT during a graceful shutdown kills workers immediately" do + addr, port = unused_port + pid = spawn_server(app: write_app, config: <<~RUBY) + listen "#{addr}:#{port}" + worker_processes 1 + RUBY + worker_pid = start_slow_request(addr, port, 30) + + Process.kill(:QUIT, pid) + assert_stderr("QUIT received, starting graceful shutdown") + Process.kill(:INT, pid) + + assert_exited(pid, 0, timeout: 2) + assert_raises(Errno::ESRCH) { Process.kill(0, worker_pid) } + ensure + Process.kill(:KILL, worker_pid) rescue nil + end + + private + + def assert_shutdown_during_startup(signal) + addr, port = unused_port + pid = spawn_server(app: write_app, config: <<~RUBY) + listen "#{addr}:#{port}" + worker_processes 1 + + module SynchronousStartup + def start(_sync = true) + super(true) + end + end + Pitchfork::HttpServer.prepend(SynchronousStartup) + + module SlowInitialWorker + def init_worker_process(worker) + $stderr.puts "initial worker pid=\#{Process.pid}" + sleep 0.01 until File.exist?("continue-initial-worker") + super + end + end + after_mold_fork { |server, _mold| server.extend(SlowInitialWorker) } + RUBY + assert_stderr(/initial worker pid=\d+/, timeout: 5) + worker_pid = Integer(read_stderr[/initial worker pid=(\d+)/, 1]) + + Process.kill(signal, pid) + mode = signal == :INT ? "immediate" : "graceful" + assert_stderr("#{signal} received, starting #{mode} shutdown", timeout: 5) + File.write("continue-initial-worker", "") + + assert_exited(pid, 0, timeout: 5) + assert_process_exits(worker_pid) + worker_pid = nil + ensure + Process.kill(:KILL, worker_pid) rescue nil + end + + def assert_process_exits(pid) + 50.times do + Process.kill(0, pid) + sleep 0.1 + rescue Errno::ESRCH + return + end + flunk "process pid=#{pid} did not exit" + end + + def spawn_pending_worker(addr, port, spawn_timeout:) + spawn_server(app: write_app, config: <<~RUBY) + listen "#{addr}:#{port}" + worker_processes 1 + spawn_timeout #{spawn_timeout} + + module SlowPendingWorker + def init_worker_process(worker) + if worker.nr == 1 + $stderr.puts "pending worker pid=\#{Process.pid}" + sleep 0.01 until File.exist?("continue-worker") + $stderr.puts "pending worker continuing" + end + super + end + end + + after_mold_fork { |server, _mold| server.extend(SlowPendingWorker) } + RUBY + end + + def write_app + File.write("slow.ru", SLOW_APP) + File.join(@pwd, "slow.ru") + end + + def start_slow_request(addr, port, seconds) + assert_healthy("http://#{addr}:#{port}") + Thread.new { http_get("http://#{addr}:#{port}/?#{seconds}", timeout: seconds + 5) rescue nil } + assert_stderr(/request started pid=\d+/, timeout: 5) + Integer(read_stderr[/request started pid=(\d+)/, 1]) + end + + def cpu_time(pid) + `ps -o time= -p #{pid}`.split(":").map(&:to_f).reverse.each_with_index.sum { |n, i| n * 60**i } + end +end