From e1a4fea797cd233f18862f792f6556a144a36c56 Mon Sep 17 00:00:00 2001 From: Volodymr Shandak <151635297+volodymyr-aloware@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:44:44 -0500 Subject: [PATCH 1/2] Share the fork server's worker handling (WorkerSet) A unit's fork server forks, admits, signals and reaps the horizon:work workers of its supervisor loop. That handling moves into WorkerSet, unchanged, so another server can fork workers the same way. UnitServer keeps the loop and passes it every worker message, as before. --- src/Runtime.php | 3 +- src/Server/UnitServer.php | 141 +++------------------------ src/Server/WorkerSet.php | 200 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 218 insertions(+), 126 deletions(-) create mode 100644 src/Server/WorkerSet.php diff --git a/src/Runtime.php b/src/Runtime.php index dce2634..8ee3d14 100644 --- a/src/Runtime.php +++ b/src/Runtime.php @@ -19,7 +19,8 @@ final class Runtime Master::class, MasterHandoff::class, Preflight::class, QueueWorkerLink::class, Runtime::class, Settings::class, Wiring::class, WorkerLink::class, Server\ChildJob::class, Server\ChildRoutine::class, Server\Opcache::class, Server\RoleGuard::class, Server\SharedLog::class, Server\Signals::class, Server\Template::class, - Server\UnitServer::class, Server\WorkServer::class, Server\WorkerContext::class, Support\Audit::class, Support\Channel::class, + Server\UnitServer::class, Server\WorkServer::class, Server\WorkerContext::class, Server\WorkerSet::class, + Support\Audit::class, Support\Channel::class, Support\CommandLine::class, Support\DotEnv::class, Support\EarlyConfig::class, Support\Fd::class, Support\Identity::class, Support\LineBufferFilter::class, Support\LineBufferedOutput::class, Support\LogTargets::class, Support\Native::class, Support\Process::class, Support\RuntimeDir::class, diff --git a/src/Server/UnitServer.php b/src/Server/UnitServer.php index c557b12..be9aeee 100644 --- a/src/Server/UnitServer.php +++ b/src/Server/UnitServer.php @@ -47,11 +47,7 @@ final class UnitServer /** @var list relays waiting for "armed" */ private array $queuedRelays = []; - /** @var array */ - private array $workers = []; - - /** @var array */ - private array $byPid = []; + private WorkerSet $workers; private float $helloElapsed = 0.0; @@ -69,6 +65,7 @@ public function __construct( $this->env = is_array($spawn['env'] ?? null) ? $spawn['env'] : []; $this->prepend = is_string($spawn['prepend'] ?? null) ? $spawn['prepend'] : null; $this->lastTick = microtime(true); + $this->workers = new WorkerSet(fn (array $message) => $this->loop?->send($message)); } public function serve(): ChildJob @@ -164,7 +161,7 @@ private function tick(): ?ChildJob $this->loop->flush(); } - $this->readWorkers(); + $this->workers->readReady(); $this->reap(); $this->watchMaster(); $this->watchHello(); @@ -196,30 +193,6 @@ private function waitForChannels(): void @stream_select($read, $write, $except, 1); } - /** A worker's startup channel carries only its `ready`; it is closed as soon as that arrived. */ - private function readWorkers(): void - { - foreach ($this->workers as $id => $worker) { - if ($worker['channel'] === null) { - continue; - } - - foreach ($worker['channel']->receive() as $message) { - if (($message['t'] ?? null) === 'ready') { - $this->workers[$id]['ready'] = true; - $this->loop?->send(['t' => 'ready', 'id' => $id]); - } - } - - $worker['channel']->flush(); - - if ($this->workers[$id]['ready'] || $worker['channel']->eof) { - $worker['channel']->close(); - $this->workers[$id]['channel'] = null; - } - } - } - /** @return list */ private function channels(): array { @@ -233,13 +206,7 @@ private function channels(): array $channels[] = $this->loop; } - foreach ($this->workers as $worker) { - if ($worker['channel'] !== null && $worker['channel']->isOpen()) { - $channels[] = $worker['channel']; - } - } - - return $channels; + return [...$channels, ...$this->workers->channels()]; } private function onUp(array $message): void @@ -278,29 +245,28 @@ private function onLoop(array $message): ?ChildJob break; case 'spawn': - return $this->forkWorker($message); + return $this->workers->fork( + $message, + $this->cwd, + array_merge(self::RELAY, [SIGCHLD]), + [$this->up, $this->loop], + $this->masterGone ? 'master gone' : null, + ); case 'admit': - if (isset($this->workers[$id]) && $this->workers[$id]['channel'] !== null) { - $this->workers[$id]['admitted'] = true; - $this->workers[$id]['channel']->send(['t' => 'admit', 'paused' => (bool) ($message['paused'] ?? false)]); - } + $this->workers->admit($id, (bool) ($message['paused'] ?? false)); break; case 'pause_state': - if (isset($this->workers[$id]) && $this->workers[$id]['channel'] !== null && ! $this->workers[$id]['ready']) { - $this->workers[$id]['channel']->send(['t' => 'pause_state', 'paused' => (bool) ($message['paused'] ?? false)]); - } + $this->workers->pauseState($id, (bool) ($message['paused'] ?? false)); break; case 'cancel': - if (isset($this->workers[$id]) && ! $this->workers[$id]['admitted']) { - posix_kill($this->workers[$id]['pid'], SIGKILL); - } + $this->workers->cancel($id); break; case 'signal': - $delivered = isset($this->workers[$id]) && posix_kill($this->workers[$id]['pid'], (int) $message['sig']); + $delivered = $this->workers->signal($id, (int) $message['sig']); $this->loop?->send(['t' => 'signalled', 'id' => $id, 'delivered' => $delivered]); break; @@ -312,69 +278,6 @@ private function onLoop(array $message): ?ChildJob return null; } - private function forkWorker(array $message): ?ChildJob - { - $id = (string) $message['id']; - - try { - $command = CommandLine::parse((string) $message['cmd'], ['horizon:work']); - } catch (Throwable $e) { - $this->loop?->send(['t' => 'spawn_failed', 'id' => $id, 'reason' => $e->getMessage()]); - - return null; - } - - if ($this->masterGone) { - $this->loop?->send(['t' => 'spawn_failed', 'id' => $id, 'reason' => 'master gone']); - - return null; - } - - $cwd = (string) ($message['cwd'] ?? $this->cwd); - $env = is_array($message['env'] ?? null) ? $message['env'] : []; - $pair = stream_socket_pair(STREAM_PF_UNIX, STREAM_SOCK_STREAM, STREAM_IPPROTO_IP); - pcntl_sigprocmask(SIG_BLOCK, array_merge(self::RELAY, [SIGCHLD]), $previous); - - try { - $pid = Process::fork(); - } catch (Throwable) { - $pid = -1; - } - - if ($pid === 0) { - @fclose($pair[0]); - $close = [$this->up, $this->loop]; - - foreach ($this->workers as $worker) { - $close[] = $worker['channel']; - } - - $prepend = is_string($message['prepend'] ?? null) ? $message['prepend'] : null; - - return ChildRoutine::become('worker', $pair[1], $close, $command, $cwd, $env, $id, prepend: $prepend); - } - - @fclose($pair[1]); - - if ($pid < 0) { - @fclose($pair[0]); - pcntl_sigprocmask(SIG_SETMASK, $previous); - $this->loop?->send(['t' => 'spawn_failed', 'id' => $id, 'reason' => 'fork failed']); - - return null; - } - - $start = Identity::startTime($pid); - $this->workers[$id] = [ - 'pid' => $pid, 'start' => $start, 'channel' => new Channel($pair[0]), 'admitted' => false, 'ready' => false, - ]; - $this->byPid[$pid] = $id; - pcntl_sigprocmask(SIG_SETMASK, $previous); - $this->loop?->send(['t' => 'spawned', 'id' => $id, 'pid' => $pid, 'start' => $start]); - - return null; - } - private function relaySignals(): void { foreach (Signals::take() as $signal => $count) { @@ -406,19 +309,7 @@ private function reap(): void $this->finish($status); } - $id = $this->byPid[$pid] ?? null; - unset($this->byPid[$pid]); - - if ($id === null) { - continue; - } - - $worker = $this->workers[$id]; - unset($this->workers[$id]); - $worker['channel']?->close(); - $this->loop?->send([ - 't' => 'exited', 'id' => $id, 'pid' => $pid, 'start' => $worker['start'], 'status' => $status, 'ready' => $worker['ready'], - ]); + $this->workers->reaped($pid, $status); } } diff --git a/src/Server/WorkerSet.php b/src/Server/WorkerSet.php new file mode 100644 index 0000000..faaa5cb --- /dev/null +++ b/src/Server/WorkerSet.php @@ -0,0 +1,200 @@ + */ + private array $workers = []; + + /** @var array */ + private array $byPid = []; + + /** @param Closure(array): void $send */ + public function __construct(private Closure $send) {} + + public function has(string $id): bool + { + return isset($this->workers[$id]); + } + + /** + * Forks the worker a `spawn` asks for. Returns the child's job in the child, null in this server. + * + * @param list $block signals held while forking, so none is handled with a half-recorded child + * @param list $close this server's own channel ends, which the worker must not keep + * @param string|null $refuse a reason to answer spawn_failed instead of forking + */ + public function fork(array $message, string $defaultCwd, array $block, array $close, ?string $refuse = null): ?ChildJob + { + $id = (string) $message['id']; + + try { + $command = CommandLine::parse((string) $message['cmd'], ['horizon:work']); + } catch (Throwable $e) { + ($this->send)(['t' => 'spawn_failed', 'id' => $id, 'reason' => $e->getMessage()]); + + return null; + } + + if ($refuse !== null) { + ($this->send)(['t' => 'spawn_failed', 'id' => $id, 'reason' => $refuse]); + + return null; + } + + $cwd = (string) ($message['cwd'] ?? $defaultCwd); + $env = is_array($message['env'] ?? null) ? $message['env'] : []; + $pair = stream_socket_pair(STREAM_PF_UNIX, STREAM_SOCK_STREAM, STREAM_IPPROTO_IP); + + if ($pair === false) { + ($this->send)(['t' => 'spawn_failed', 'id' => $id, 'reason' => 'socket pair']); + + return null; + } + + pcntl_sigprocmask(SIG_BLOCK, $block, $previous); + + try { + $pid = Process::fork(); + } catch (Throwable) { + $pid = -1; + } + + if ($pid === 0) { + @fclose($pair[0]); + + foreach ($this->workers as $worker) { + $close[] = $worker['channel']; + } + + $prepend = is_string($message['prepend'] ?? null) ? $message['prepend'] : null; + + return ChildRoutine::become('worker', $pair[1], $close, $command, $cwd, $env, $id, prepend: $prepend); + } + + @fclose($pair[1]); + + if ($pid < 0) { + @fclose($pair[0]); + pcntl_sigprocmask(SIG_SETMASK, $previous); + ($this->send)(['t' => 'spawn_failed', 'id' => $id, 'reason' => 'fork failed']); + + return null; + } + + $start = Identity::startTime($pid); + $this->workers[$id] = [ + 'pid' => $pid, 'start' => $start, 'channel' => new Channel($pair[0]), 'admitted' => false, 'ready' => false, + ]; + $this->byPid[$pid] = $id; + pcntl_sigprocmask(SIG_SETMASK, $previous); + ($this->send)(['t' => 'spawned', 'id' => $id, 'pid' => $pid, 'start' => $start]); + + return null; + } + + public function admit(string $id, bool $paused): void + { + if (isset($this->workers[$id]) && $this->workers[$id]['channel'] !== null) { + $this->workers[$id]['admitted'] = true; + $this->workers[$id]['channel']->send(['t' => 'admit', 'paused' => $paused]); + } + } + + /** Before its first loop a worker has no signal handlers: a pause reaches it on its startup channel. */ + public function pauseState(string $id, bool $paused): void + { + if (isset($this->workers[$id]) && $this->workers[$id]['channel'] !== null && ! $this->workers[$id]['ready']) { + $this->workers[$id]['channel']->send(['t' => 'pause_state', 'paused' => $paused]); + } + } + + /** A worker not admitted yet has run no app code: nothing to let finish. */ + public function cancel(string $id): void + { + if (isset($this->workers[$id]) && ! $this->workers[$id]['admitted']) { + posix_kill($this->workers[$id]['pid'], SIGKILL); + } + } + + public function signal(string $id, int $signal): bool + { + return isset($this->workers[$id]) && posix_kill($this->workers[$id]['pid'], $signal); + } + + /** A worker's startup channel carries only its `ready`; it is closed as soon as that arrived. */ + public function readReady(): void + { + foreach ($this->workers as $id => $worker) { + if ($worker['channel'] === null) { + continue; + } + + foreach ($worker['channel']->receive() as $message) { + if (($message['t'] ?? null) === 'ready') { + $this->workers[$id]['ready'] = true; + ($this->send)(['t' => 'ready', 'id' => $id]); + } + } + + $worker['channel']->flush(); + + if ($this->workers[$id]['ready'] || $worker['channel']->eof) { + $worker['channel']->close(); + $this->workers[$id]['channel'] = null; + } + } + } + + /** @return list the startup channels still open, for the server's select */ + public function channels(): array + { + $channels = []; + + foreach ($this->workers as $worker) { + if ($worker['channel'] !== null && $worker['channel']->isOpen()) { + $channels[] = $worker['channel']; + } + } + + return $channels; + } + + /** Whether $pid was one of these workers; if so, its exit has been reported. */ + public function reaped(int $pid, int $status): bool + { + $id = $this->byPid[$pid] ?? null; + + if ($id === null) { + return false; + } + + unset($this->byPid[$pid]); + $worker = $this->workers[$id]; + unset($this->workers[$id]); + $worker['channel']?->close(); + ($this->send)([ + 't' => 'exited', 'id' => $id, 'pid' => $pid, 'start' => $worker['start'], 'status' => $status, 'ready' => $worker['ready'], + ]); + + return true; + } + + /** @return list */ + public function pids(): array + { + return array_keys($this->byPid); + } +} From 185c17d6286e592ba4396b2f440b203dfcb84d9a Mon Sep 17 00:00:00 2001 From: Volodymr Shandak <151635297+volodymyr-aloware@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:44:44 -0500 Subject: [PATCH 2/2] Let the template fork workers for supervisors inside the master Some apps run Horizon's supervisors inside the master process instead of one process each. Their workers can now be forked as well: the master asks its template for each worker (a spawn with role: worker), and the template forks it through WorkerSet, with no fork server in between. The template TERMs the workers it forked when its master is gone, so none outlives it. On the master's side, Master::start() in inline mode (queue-prefork.horizon_inline) skips the check for the app's own AddSupervisor binding, applies the off switches itself and pumps on MasterSupervisorLooped. Master::workerProcess() hands out forked handles with a crash breaker per supervisor, and ForksWorkersFromTemplate lets the app's process pool use them. Horizon restarts a worker on the handle it has, so a stopped handle whose template is gone, or whose supervisor the breaker turned stock, is replaced with a new one, forked again or stock. The doctor reads horizon_inline as well. --- CHANGELOG.md | 4 + README.md | 9 ++ config/queue-prefork.php | 5 ++ src/Client/ForkedProcess.php | 7 ++ src/Console/DoctorCommand.php | 3 +- src/Horizon/ForksWorkersFromTemplate.php | 59 +++++++++++++ src/Master.php | 102 +++++++++++++++++++++- src/Preflight.php | 14 ++- src/Server/Template.php | 71 ++++++++++++++- tests/Fixtures/fake-worker.php | 9 ++ tests/Mechanics/TemplateHarness.php | 14 +++ tests/Mechanics/TemplateWorkersTest.php | 105 +++++++++++++++++++++++ 12 files changed, 392 insertions(+), 10 deletions(-) create mode 100644 src/Horizon/ForksWorkersFromTemplate.php create mode 100644 tests/Mechanics/TemplateWorkersTest.php diff --git a/CHANGELOG.md b/CHANGELOG.md index 173355e..bc4855b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ All notable changes to this project are documented here. The project uses [Semantic Versioning](https://semver.org/). +## [Unreleased] + +- **Supervisors inside the master:** an app that runs Horizon's supervisors in the master process can have the template fork their workers directly (`horizon_inline`, `Master::start()`, `ForksWorkersFromTemplate`). The template's worker handling is shared with the fork servers (`WorkerSet`), and the template TERMs the workers it forked when its master is gone. + ## [1.1.2] - 2026-11-20 - **Stack traces in Flare, Sentry and Nightwatch:** a forked worker's stack trace ends in the launcher the template was started from, not in `artisan`. The launcher was a copy outside `vendor/`, so error trackers counted it as the app's own code. Flare drops every frame above the first frame of the app's code, so for a job failing in vendor code (a package's job, for example) only the launcher was left. Sentry marked it as the app's code, and Nightwatch placed such exceptions there. Templates now start through a link to the package's own `bin/template.php`, so traces name it in `vendor/`, and error trackers see the trace as in stock. Reported by Funsho Arashi (#5). diff --git a/README.md b/README.md index 5b8f88a..320bb26 100644 --- a/README.md +++ b/README.md @@ -153,6 +153,15 @@ Each `queue:work` process hands itself to a shared template before Laravel boots - **Horizon works as usual,** because supervisor and worker code is Horizon's own: the dashboard, `horizon:terminate`, `pause`, `continue`, `purge`, autoscaling, and supervisord's `stopasgroup` and `killasgroup`. - **Fallbacks.** A failed check runs the stock master in the same pid. If the template dies on Linux, Horizon exits and your process manager restarts it. A supervisor whose workers keep dying before their first loop is replaced by a stock one; a supervisor and its workers are always all forked or all stock. Each fallback logs a `queue-prefork:` error line and fires an event. +### Supervisors inside the master + +Some apps run Horizon's supervisors inside the master instead of one process each. Their workers can still be forked: the template forks each worker itself, on the master's request, with no fork server in between. + +- **Turn it on** with `QUEUE_PREFORK_HORIZON_INLINE=true` (config `horizon_inline`), which the doctor reads too. +- **In the app's own `horizon` command,** call `app(Tevkert\QueuePrefork\Master::class)->start()` before the provisioning plan deploys, when the container has that class bound (the package binds it only for `artisan horizon`). It returns false and changes nothing when the package is switched off or a check fails. +- **Give each in-process supervisor a process pool that uses `Tevkert\QueuePrefork\Horizon\ForksWorkersFromTemplate`.** Its workers are forked while the master runs this way and exec'd as stock otherwise. A stopped worker whose template is gone, or whose supervisor the crash breaker turned stock, gets a new handle, forked again or stock, instead of being restarted on the old one. +- **Don't run `horizon:purge` in this mode.** It looks for workers among the children of supervisor processes, and these workers are the template's children. + ### Fork safety Before forking, a template audits itself: sockets, epoll, eventfd or extra threads that extensions open at startup mean stock workers. ddtrace, Elastic APM and gRPC with fork support (`grpc.enable_fork_support=1`, which rebuilds its threads and poller in each fork) are allowed; workers run stock with New Relic, gRPC without fork support, datadog-profiling and Blackfire. diff --git a/config/queue-prefork.php b/config/queue-prefork.php index 59985bb..0fd4085 100644 --- a/config/queue-prefork.php +++ b/config/queue-prefork.php @@ -10,6 +10,11 @@ 'queue_work' => env('QUEUE_PREFORK_QUEUE_WORK', true), 'horizon' => env('QUEUE_PREFORK_HORIZON', true), + // The app runs Horizon's supervisors inside the master and asks the template only for their workers + // (Master::start(inline: true), ForksWorkersFromTemplate). Its own AddSupervisor binding is then expected; + // the doctor checks this mode instead of the forked supervisors. + 'horizon_inline' => env('QUEUE_PREFORK_HORIZON_INLINE', false), + // The templates' opcache. Its shared memory holds the compiled code of the whole app. 'opcache' => [ 'memory_consumption' => env('QUEUE_PREFORK_OPCACHE_MB', 192), diff --git a/src/Client/ForkedProcess.php b/src/Client/ForkedProcess.php index 7f02ac7..e5427da 100644 --- a/src/Client/ForkedProcess.php +++ b/src/Client/ForkedProcess.php @@ -135,6 +135,7 @@ public function start(?callable $callback = null, array $env = []): void $this->broker->request("spawn:{$this->id}", [ 't' => 'spawn', 'id' => $this->id, + 'role' => $this->role, 'cmd' => $this->line, 'cwd' => $this->directory, 'env' => getenv(), @@ -216,6 +217,12 @@ public function ready(): void } } + /** Whether the server this handle forks through is gone: starting it again can only fail. */ + public function serverGone(): bool + { + return $this->broker->lost; + } + /** @internal */ public function serverLost(): void { diff --git a/src/Console/DoctorCommand.php b/src/Console/DoctorCommand.php index 48cc0e0..4f2af8f 100644 --- a/src/Console/DoctorCommand.php +++ b/src/Console/DoctorCommand.php @@ -28,7 +28,8 @@ public function handle(): int $horizon = class_exists(MasterSupervisor::class); $release = realpath(base_path()) ?: base_path(); $pidOne = PHP_OS_FAMILY === 'Linux' ? str_replace("\0", ' ', (string) @file_get_contents('/proc/1/cmdline')) : ''; - $reasons = $horizon ? [...Switches::off($release, 'horizon'), ...Preflight::check($this->laravel), ...self::pidOne($pidOne, 'horizon')] : []; + $inline = (bool) $this->laravel['config']->get('queue-prefork.horizon_inline', false); + $reasons = $horizon ? [...Switches::off($release, 'horizon'), ...Preflight::check($this->laravel, inline: $inline), ...self::pidOne($pidOne, 'horizon')] : []; $queueWork = self::queueWork($release, self::pidOne($pidOne, 'queue:work')); $warnings = [...self::hostWarnings($horizon), ...$this->earlyMismatches($release)]; diff --git a/src/Horizon/ForksWorkersFromTemplate.php b/src/Horizon/ForksWorkersFromTemplate.php new file mode 100644 index 0000000..c881d04 --- /dev/null +++ b/src/Horizon/ForksWorkersFromTemplate.php @@ -0,0 +1,59 @@ +preforkMaster()?->workerProcess($this->options); + + return $process === null ? parent::createProcess() : new WorkerProcess($process); + } + + public function monitor() + { + $this->replaceRetiredWorkers(); + + parent::monitor(); + } + + /** + * Horizon restarts a stopped worker on the handle it has. A forked handle whose template is gone, or whose + * supervisor the crash breaker turned stock, gets a new one instead: forked again, or stock. + */ + protected function replaceRetiredWorkers(): void + { + $master = $this->preforkMaster(); + + foreach ($this->processes as $index => $worker) { + $process = $worker->process; + + if (! $process instanceof ForkedProcess || $process->isRunning()) { + continue; + } + + if ($process->serverGone() || $master?->retires($process, $this->options->name)) { + $this->processes[$index] = $this->createProcess()->handleOutputUsing(function ($type, $line) { + call_user_func($this->output, $type, $line); + }); + } + } + } + + private function preforkMaster(): ?Master + { + $app = app(); + + return $app->bound(Master::class) ? $app->make(Master::class) : null; + } +} diff --git a/src/Master.php b/src/Master.php index 46ea30a..3bfedec 100644 --- a/src/Master.php +++ b/src/Master.php @@ -4,6 +4,7 @@ use Illuminate\Contracts\Foundation\Application; use Illuminate\Support\Facades\Redis; +use Laravel\Horizon\Events\MasterSupervisorLooped; use Laravel\Horizon\MasterSupervisor; use Laravel\Horizon\SupervisorOptions; use Tevkert\QueuePrefork\Client\Broker; @@ -75,20 +76,40 @@ final class Master /** @var array supervisor name => current unit handle */ private array $units = []; + /** @var array> supervisor name => recent exits of its workers before their first loop */ + private array $earlyWorkerExits = []; + public function __construct(private Application $app) {} - public function start(?callable $warn = null): bool + /** + * @param bool|null $inline the app runs its supervisors in this process and asks for their workers through + * workerProcess(), so its own AddSupervisor binding is expected, not a conflict; + * null reads queue-prefork.horizon_inline, which the doctor reads too + */ + public function start(?callable $warn = null, ?bool $inline = null): bool { Runtime::preload(); + $inline ??= (bool) $this->app['config']->get('queue-prefork.horizon_inline', false); + + if ($inline && ($off = $this->inlineOff()) !== null) { + $this->reason = $off; + + return false; + } try { - $reasons = Preflight::check($this->app); + $reasons = Preflight::check($this->app, inline: $inline); if ($reasons === [] && $this->startTemplate()) { $this->active = true; $this->mode = 'prefork'; Report::event('PreforkStarted', ['template' => $this->template['pid'] ?? null], 'info'); + if ($inline) { + // The app's own horizon command runs the loop: nothing else registers this. + $this->app['events']->listen(MasterSupervisorLooped::class, fn () => $this->tick()); + } + return true; } @@ -113,6 +134,21 @@ public function start(?callable $warn = null): bool return false; } + /** + * Why an inline start stays stock: Wiring checks the same switches before it binds the package's own + * command, which an app running its supervisors in the master replaces with its own. + */ + private function inlineOff(): ?string + { + $off = Support\Switches::off(base_path(), 'horizon'); + + if ($off !== []) { + return implode('; ', $off); + } + + return Settings::enabled($this->app, 'horizon') ? null : 'switched off ('.Settings::switchSource($this->app, 'horizon').')'; + } + /** Called on every MasterSupervisorLooped (and so also while paused). */ public function tick(): void { @@ -158,6 +194,68 @@ public function unitProcess(SupervisorOptions $options): ?ForkedProcess return $this->units[$options->name] = $unit; } + /** + * A worker forked by the template for a supervisor that runs in this process, or null for a stock one: + * when nothing forks, for a command line the template does not run, or once the supervisor's workers kept + * exiting before their first loop. A handle once forked stays forked when its worker restarts. + */ + public function workerProcess(SupervisorOptions $options): ?ForkedProcess + { + if (! $this->canForkUnits() || isset($this->stockNames[$options->name])) { + return null; + } + + $line = $options->toWorkerCommand(); + + try { + CommandLine::parse($line, ['horizon:work']); + } catch (Throwable $e) { + $this->stockNames[$options->name] = $e->getMessage(); + + return null; + } + + $worker = new ForkedProcess($this->broker, 'worker', $line, $options->directory ?? base_path(), ! config('horizon.fast_termination')); + $worker->onExit = function (ForkedProcess $process, int $status, bool $wasReady) use ($options): void { + $this->workerExited($options->name, $process, $status, $wasReady); + }; + + return $worker; + } + + /** + * Whether a stopped forked worker of $supervisor needs a new handle: Horizon restarts a worker on the + * handle it has, which still forks through a template that is gone, or for a supervisor now stock. + */ + public function retires(ForkedProcess $worker, string $supervisor): bool + { + return $worker->serverGone() || isset($this->stockNames[$supervisor]); + } + + /** The unit breaker of Link::workerExited(), kept here per supervisor, as no loop of its own runs it. */ + private function workerExited(string $supervisor, ForkedProcess $process, int $status, bool $wasReady): void + { + if ($wasReady || $status === 0 || $process->parentSignalled || in_array($status, [137, 75, 72], true)) { + return; + } + + $now = microtime(true); + + if ($now - $process->spawnedAt > 120) { + return; + } + + $window = (float) Runtime::config('breaker_window', 300); + $exits = array_values(array_filter($this->earlyWorkerExits[$supervisor] ?? [], fn (float $at) => $now - $at < $window)); + $exits[] = $now; + $this->earlyWorkerExits[$supervisor] = $exits; + + if (count($exits) >= (int) Runtime::config('breaker_exits', 5)) { + $this->stockNames[$supervisor] = 'crash breaker'; + Report::event('BreakerTripped', ['scope' => 'supervisor', 'supervisor' => $supervisor, 'status' => $status]); + } + } + public function statusDir(): string { return $this->statusDir; diff --git a/src/Preflight.php b/src/Preflight.php index a472a63..ba0ccde 100644 --- a/src/Preflight.php +++ b/src/Preflight.php @@ -19,8 +19,11 @@ */ final class Preflight { - /** @return list */ - public static function check(Application $app, bool $probes = true): array + /** + * @param bool $inline the app's supervisors run in the master, which forks only their workers + * @return list + */ + public static function check(Application $app, bool $probes = true, bool $inline = false): array { $reasons = []; $config = (array) $app['config']->get('queue-prefork', []); @@ -53,7 +56,12 @@ public static function check(Application $app, bool $probes = true): array $horizon = self::horizonVersion(); // The bindings and command strings are Horizon's too: read them only from a Horizon that has them. - $reasons = array_merge($reasons, $horizon, $horizon === [] ? [...self::bindings($app), ...self::commandStrings($app)] : [], self::extensions($config)); + $reasons = array_merge( + $reasons, + $horizon, + $horizon === [] ? [...($inline ? [] : self::bindings($app)), ...self::commandStrings($app)] : [], + self::extensions($config), + ); if ($probes && $reasons === []) { $reasons = array_merge($reasons, self::probes($app)); diff --git a/src/Server/Template.php b/src/Server/Template.php index 0dcca98..1b6c355 100644 --- a/src/Server/Template.php +++ b/src/Server/Template.php @@ -61,6 +61,9 @@ final class Template /** @var array pid => unit id */ private array $byPid = []; + /** Workers forked for a master that runs its supervisors in its own process (`role: worker` spawns). */ + private WorkerSet $workers; + private float $lastOpcacheReport; public static function main(): ChildJob @@ -318,6 +321,7 @@ private function __construct(?Channel $control = null) } $this->lastOpcacheReport = microtime(true); + $this->workers = new WorkerSet(fn (array $message) => $this->control->send($message)); } private function hosting(): bool @@ -354,11 +358,13 @@ private function serve(): ChildJob $unit['channel']->flush(); } + $this->workers->readReady(); $this->control->flush(); $this->reap(); // A host ends when it reaps its master (reap()); a spawned template when its master is gone. if (! $this->hosting() && ($this->control->eof || ! $this->masterAlive())) { + $this->stopWorkers(); RoleGuard::exitServer(0); } @@ -397,6 +403,14 @@ private function waitForStreams(): void } } + foreach ($this->workers->channels() as $channel) { + $read[] = $channel->stream(); + + if ($channel->wantsWrite()) { + $write[] = $channel->stream(); + } + } + if ($read === [] && $write === []) { usleep(200000); @@ -425,17 +439,32 @@ private function onControl(array $message): ?ChildJob switch ($message['t'] ?? null) { case 'spawn': - return $this->forkUnit($message); + return ($message['role'] ?? 'unit') === 'worker' ? $this->forkWorker($message) : $this->forkUnit($message); case 'admit': + if ($this->workers->has($id)) { + $this->workers->admit($id, (bool) ($message['paused'] ?? false)); + } elseif (isset($this->units[$id])) { + $this->units[$id]['channel']->send(['t' => 'admit', 'id' => $id]); + } + break; + + case 'pause_state': + $this->workers->pauseState($id, (bool) ($message['paused'] ?? false)); + break; + case 'cancel': - if (isset($this->units[$id])) { - $this->units[$id]['channel']->send(['t' => $message['t'], 'id' => $id]); + if ($this->workers->has($id)) { + $this->workers->cancel($id); + } elseif (isset($this->units[$id])) { + $this->units[$id]['channel']->send(['t' => 'cancel', 'id' => $id]); } break; case 'signal': - $delivered = $this->signalUnit($id, (int) $message['sig'], ($message['target'] ?? 'one') === 'both'); + $delivered = $this->workers->has($id) + ? $this->workers->signal($id, (int) $message['sig']) + : $this->signalUnit($id, (int) $message['sig'], ($message['target'] ?? 'one') === 'both'); $this->control->send(['t' => 'signalled', 'id' => $id, 'delivered' => $delivered]); break; @@ -525,6 +554,10 @@ private function forkUnit(array $message): ?ChildJob $unit['channel']->close(); } + foreach ($this->workers->channels() as $channel) { + $channel->close(); + } + pcntl_sigprocmask(SIG_SETMASK, $previous); return (new UnitServer(new Channel($pair[1]), $message, $command))->serve(); @@ -553,6 +586,30 @@ private function forkUnit(array $message): ?ChildJob return null; } + /** A worker for a master whose supervisors run in its own process: forked here, with no fork server between. */ + private function forkWorker(array $message): ?ChildJob + { + $close = [$this->control]; + + foreach ($this->units as $unit) { + $close[] = $unit['channel']; + } + + // A relayed signal arriving mid-fork stays pending until the child has reset its handlers. + return $this->workers->fork($message, (string) getcwd(), [SIGCHLD, ...self::RELAY], $close); + } + + /** + * The workers this template forked outlive it otherwise: no supervisor loop is left to stop them. + * TERM lets each finish its current job, as a stock supervisor's terminate does. + */ + private function stopWorkers(): void + { + foreach ($this->workers->pids() as $pid) { + posix_kill($pid, SIGTERM); + } + } + private function onUnitMessage(string $id, array $message): void { switch ($message['t'] ?? null) { @@ -582,6 +639,10 @@ private function reap(): void $this->masterExited($raw); // its units drain on their own, as under a stock master } + if ($this->workers->reaped($pid, Signals::status($raw))) { + continue; + } + $id = $this->byPid[$pid] ?? null; unset($this->byPid[$pid]); @@ -604,6 +665,8 @@ private function reap(): void */ private function masterExited(int $raw): never { + $this->stopWorkers(); + if ($this->stockAfterMaster === null || ! pcntl_wifexited($raw) || pcntl_wexitstatus($raw) !== 0) { self::exitLike($raw); } diff --git a/tests/Fixtures/fake-worker.php b/tests/Fixtures/fake-worker.php index f8e77d9..94c41ec 100644 --- a/tests/Fixtures/fake-worker.php +++ b/tests/Fixtures/fake-worker.php @@ -5,11 +5,16 @@ use Tevkert\QueuePrefork\Runtime; $exit = 0; +$hold = 0; foreach ($argv as $arg) { if (str_starts_with($arg, '--exit=')) { $exit = (int) substr($arg, 7); } + + if (str_starts_with($arg, '--hold=')) { + $hold = (int) substr($arg, 7); + } } Runtime::$channel->send(['t' => 'ready', 'id' => Runtime::$requestId]); @@ -29,4 +34,8 @@ 'stdin_ok' => is_resource(STDIN) && @stream_isatty(STDIN) === false, ])."\n", FILE_APPEND | LOCK_EX); +if ($hold > 0) { + sleep($hold); // a signal with its default action ends it here +} + exit($exit); diff --git a/tests/Mechanics/TemplateHarness.php b/tests/Mechanics/TemplateHarness.php index e92a0f7..c948534 100644 --- a/tests/Mechanics/TemplateHarness.php +++ b/tests/Mechanics/TemplateHarness.php @@ -87,6 +87,20 @@ public function spawnUnit(string $id, string $options, array $env = []): void ]); } + /** A worker for a master that runs its supervisors in its own process: the template forks it directly. */ + public function spawnWorker(string $id, string $options, string $command = 'horizon:work'): void + { + $worker = dirname(__DIR__).'/Fixtures/fake-worker.php'; + $this->channel->send([ + 't' => 'spawn', + 'id' => $id, + 'role' => 'worker', + 'cmd' => "exec php {$worker} {$command} redis --supervisor=test:inline {$options}", + 'cwd' => dirname(__DIR__, 2), + 'env' => ['HP_TEST_OUT' => $this->out, 'HP_TEST_WORKER_ENV' => 'from-master'] + getenv(), + ]); + } + /** Waits for the next message of the given type (others are kept in $seen). */ public function await(string $type, float $timeout = 10, ?string $id = null): ?array { diff --git a/tests/Mechanics/TemplateWorkersTest.php b/tests/Mechanics/TemplateWorkersTest.php new file mode 100644 index 0000000..e3555b2 --- /dev/null +++ b/tests/Mechanics/TemplateWorkersTest.php @@ -0,0 +1,105 @@ +markTestSkipped('needs pcntl'); + } + } + + protected function tearDown(): void + { + $this->t?->close(); + } + + public function test_the_template_forks_a_worker_for_its_master_and_reports_its_lifecycle(): void + { + $this->t = new TemplateHarness; + $this->t->spawnWorker('w1', '--exit=5'); + + $spawned = $this->t->await('spawned', 10, 'w1'); + $this->assertNotNull($spawned); + $this->t->channel->send(['t' => 'admit', 'id' => 'w1', 'paused' => false]); + + $this->assertNotNull($this->t->await('ready', 10, 'w1'), 'the worker\'s first loop reaches the master'); + $exited = $this->t->await('exited', 10, 'w1'); + $this->assertSame(5, $exited['status'] ?? null); + $this->assertTrue($exited['ready']); + $this->assertSame($spawned['pid'], $exited['pid']); + + [$worker] = $this->t->events('worker'); + $this->assertSame($spawned['pid'], $worker['pid']); + $this->assertSame($this->t->ready['pid'], $worker['ppid'], 'the template itself is the worker\'s parent'); + $this->assertSame('horizon:work', $worker['argv'][1]); + $this->assertSame('from-master', $worker['env'], 'the worker gets the requester\'s environment'); + $this->assertTrue($worker['opcache'], 'the worker shares the template\'s opcache'); + $this->assertTrue($worker['stdin_ok']); + } + + public function test_the_template_relays_a_signal_to_its_worker(): void + { + $this->t = new TemplateHarness; + $this->t->spawnWorker('w2', '--hold=30'); + $this->assertNotNull($this->t->await('spawned', 10, 'w2')); + $this->t->channel->send(['t' => 'admit', 'id' => 'w2', 'paused' => false]); + $this->assertNotNull($this->t->await('ready', 10, 'w2')); + + $this->t->channel->send(['t' => 'signal', 'id' => 'w2', 'sig' => SIGTERM]); + + $this->assertTrue($this->t->await('signalled', 10, 'w2')['delivered'] ?? false); + $this->assertSame(128 + SIGTERM, $this->t->await('exited', 10, 'w2')['status'] ?? null); + } + + public function test_the_template_forks_only_horizon_workers_as_workers(): void + { + $this->t = new TemplateHarness; + $this->t->spawnWorker('w3', '--exit=0', 'horizon:supervisor'); + + $failed = $this->t->await('spawn_failed', 10, 'w3'); + $this->assertNotNull($failed); + $this->assertSame([], $this->t->events('worker'), 'nothing ran'); + } + + public function test_the_template_stops_its_workers_when_the_master_is_gone(): void + { + $master = proc_open([PHP_BINARY, '-r', 'sleep(60);'], [], $pipes); + $masterPid = proc_get_status($master)['pid']; + usleep(200000); + $this->t = new TemplateHarness([], $masterPid); + $this->t->spawnWorker('w4', '--hold=30'); + $spawned = $this->t->await('spawned', 10, 'w4'); + $this->t->channel->send(['t' => 'admit', 'id' => 'w4', 'paused' => false]); + $this->assertNotNull($this->t->await('ready', 10, 'w4')); + + posix_kill($masterPid, SIGKILL); + proc_close($master); + + $deadline = microtime(true) + 15; + + while (microtime(true) < $deadline && self::running($spawned['pid'])) { + usleep(100000); + } + + $this->assertFalse(self::running($spawned['pid']), 'the worker got TERM instead of outliving the template'); + } + + /** Running, not a zombie: an orphan waits to be reaped by whatever PID 1 is. */ + private static function running(int $pid): bool + { + $stat = @file_get_contents("/proc/{$pid}/stat"); + + return $stat !== false && preg_match('/\) ([A-Z]) /', $stat, $m) === 1 && $m[1] !== 'Z'; + } +}