Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
# Unreleased

- Fix the monitor using a full CPU core while it waits for workers during a graceful shutdown.
- Fix a second `QUIT` or `TERM` during a graceful shutdown making the monitor exit before pending or running workers.
- Fix `TTIN` during a graceful shutdown spawning new workers.
- Prevent `USR1` from restarting the monitor during a graceful shutdown.
- Honor shutdown signals while the server is starting.
- Preserve worker spawn deadlines when the mold starts workers.

# 0.18.4
Expand Down
63 changes: 36 additions & 27 deletions lib/pitchfork/http_server.rb
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@ def initialize(app, options = {})
@exit_status = 0
@app = app
@respawn = false
@graceful_shutdown = true
@refork_max_unavailable = nil
@consecutive_spawn_errors = 0
@max_consecutive_spawn_errors = nil
Expand Down Expand Up @@ -226,7 +227,7 @@ def start(sync = true)

if REFORKING_AVAILABLE
spawn_initial_mold
wait_for_pending_workers
return self if wait_for_pending_workers == StopIteration
unless @children.mold
raise BootFailure, "The initial mold failed to boot"
end
Expand All @@ -241,7 +242,7 @@ def start(sync = true)
# We could just return here as we'd register them later in #join.
# However a good part of the test suite assumes #start only return
# once all initial workers are spawned.
wait_for_pending_workers
return self if wait_for_pending_workers == StopIteration
end

@after_monitor_ready&.call(self)
Expand Down Expand Up @@ -324,7 +325,7 @@ def listen(address, opt = listener_opts[address] || {})
# one-at-a-time time and we'll happily drop signals in case somebody
# is signalling us too often.
def join
@respawn = true
@respawn = !SharedMemory.shutting_down?

proc_name role: 'monitor', status: ARGV.join(' ')

Expand All @@ -336,7 +337,7 @@ def join
end
@ready_pipe = @ready_pipe.close rescue nil
end
while true
until SharedMemory.shutting_down?
begin
if monitor_loop == StopIteration
break
Expand All @@ -345,7 +346,7 @@ def join
Pitchfork.log_error(@logger, "monitor loop error", e)
end
end
stop # gracefully shutdown all workers on our way out
stop
logger.info "monitor complete status=#{@exit_status}"
@exit_status
end
Expand Down Expand Up @@ -376,8 +377,7 @@ def monitor_loop(sleep = true)
if @max_consecutive_spawn_errors && @consecutive_spawn_errors > @max_consecutive_spawn_errors && !SharedMemory.shutting_down?
logger.fatal("#{@consecutive_spawn_errors} consecutive failures to spawn children, aborting - broken after_worker_fork callback?")
@exit_status = 1
SharedMemory.shutting_down!
stop(false)
request_shutdown(false)
return StopIteration
end
else
Expand All @@ -393,25 +393,24 @@ def monitor_loop(sleep = true)

def monitor_handle_message(message)
case message
when :QUIT, :TERM # graceful shutdown
SharedMemory.shutting_down!
logger.info "#{message} received, starting graceful shutdown"
return StopIteration
when :INT # immediate shutdown
SharedMemory.shutting_down!
logger.info "#{message} received, starting immediate shutdown"
stop(false)
when :QUIT, :TERM, :INT
graceful = message != :INT
request_shutdown(graceful)
logger.info "#{message} received, starting #{graceful ? 'graceful' : 'immediate'} shutdown"
return StopIteration
when :USR1 # trigger a hot restart
trigger_restart
if SharedMemory.shutting_down?
logger.error "Can't trigger a restart as the server is shutting down"
else
trigger_restart
end
when :USR2 # trigger a promotion
if @respawn
trigger_refork
else
logger.error "Can't trigger a refork as the server is shutting down"
end
when :TTIN
@respawn = true
self.worker_processes += 1
when :TTOU
self.worker_processes -= 1 if self.worker_processes > 0
Expand Down Expand Up @@ -461,20 +460,19 @@ def monitor_handle_message(message)
def stop(graceful = true)
proc_name role: 'monitor', status: 'shutting down'
@respawn = false
SharedMemory.shutting_down!
wait_for_pending_workers
request_shutdown(graceful)
wait_for_pending_workers(stop_on_signal: false)
Pitchfork.close_listeners

limit = Pitchfork.time_now + timeout
until @children.empty? || Pitchfork.time_now > limit
if graceful
if @graceful_shutdown
@children.soft_kill_all(:TERM)
else
@children.hard_kill_all(:INT)
end
if monitor_loop(false) == StopIteration
return StopIteration
end
monitor_sleep(0.1)
monitor_loop(false)
end

@children.each do |child|
Expand Down Expand Up @@ -528,6 +526,11 @@ def check_client_connection=(bool)

private

def request_shutdown(graceful)
@graceful_shutdown = false unless graceful
SharedMemory.shutting_down!
end

def refork_max_unavailable
@refork_max_unavailable ||= (worker_processes * 0.1).ceil
end
Expand Down Expand Up @@ -876,10 +879,10 @@ def spawn_missing_workers
exit!
end

def wait_for_pending_workers
def wait_for_pending_workers(stop_on_signal: true)
while @children.pending_workers?
monitor_sleep(0.5)
if monitor_loop(false) == StopIteration
if monitor_loop(false) == StopIteration && stop_on_signal
return StopIteration
end
end
Expand Down Expand Up @@ -1073,14 +1076,20 @@ def reset_signal_handlers
[:QUIT, :TERM, :INT].each { |sig| trap(sig) { exit!(0) } }
end

def register_child_and_exit_if_shutting_down(child)
# Register first so the monitor can account for the child during shutdown.
child.register_to_monitor(@control_socket[1])
exit!(0) if SharedMemory.shutting_down?
end

# gets rid of stuff the worker has no business keeping track of
# to free some resources and drops all sig handlers.
# traps for USR2, and HUP may be set in the after_worker_fork/after_mold_fork Procs
# by the user.
def init_worker_process(worker)
proc_name role: "(#{worker.version}.#{worker.generation}) worker[#{worker.nr}]", status: "init"
worker.reset
worker.register_to_monitor(@control_socket[1])
register_child_and_exit_if_shutting_down(worker)
# we'll re-trap :QUIT and :TERM later for graceful shutdown iff we accept clients
exit_sigs = [ :QUIT, :TERM, :INT ]
exit_sigs.each { |sig| trap(sig) { exit!(0) } }
Expand All @@ -1106,7 +1115,7 @@ def init_worker_process(worker)
def init_service_process(service)
proc_name role: "(#{service.version}.#{service.generation}) service", status: "init"
Pitchfork.close_listeners # Don't appear as listening to incoming requests
service.register_to_monitor(@control_socket[1])
register_child_and_exit_if_shutting_down(service)
readers = [service]
trap(:QUIT) { nuke_listeners!(readers) }
trap(:TERM) { nuke_listeners!(readers) }
Expand Down
Loading
Loading