Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions docker/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ After changing an env switch, recreate: `docker compose up -d --force-recreate a

```sh
docker compose exec app-stock php artisan fixtures:dispatch --list
docker compose exec app-stock php artisan fixtures:dispatch # all but crashes and calls-artisan
docker compose exec app-stock php artisan fixtures:dispatch # all but crashes, calls-artisan and fails-in-vendor
docker compose exec app-stock php artisan fixtures:dispatch crashes # exit(255); --crash=segv for SIGSEGV
docker compose exec app-stock php artisan fixtures:dispatch sleeps --sleep=20 --queue=import
docker compose exec app-stock php artisan fixtures:dispatch sleeps-steady --sleep=20 # sleeps through signals
Expand Down Expand Up @@ -215,5 +215,8 @@ heavy jobs (image renditions, LLM calls, Livewire renders, bulk maintenance):
Forge server. One override, `php/conf.d/zz-cli.ini`: `memory_limit=-1`, as Ubuntu's CLI php.ini has.
Otherwise a worker would die at 128M before Horizon's own `memory` check (128 MB) recycles it.
- `workbench/composer.json` pins `laravel/horizon` to `5.52.*`.
- The package is a Composer path repository (`..`, symlinked), so changes to `src/` need no reinstall;
restart Horizon (`docker compose restart app-prefork`) to load them.
- The package is a Composer path repository (`..`), but `setup` replaces Composer's symlink with a copy
of `bin/`, `config/`, `src/` and `composer.json`, as an install from Packagist has: PHP resolves
symlinks in stack traces, and error trackers judge a frame by its path. `setup` copies again on every
`up`, so recreate the stacks (`docker compose up -d --force-recreate app-stock app-prefork`) to load
changes to `src/`.
20 changes: 18 additions & 2 deletions docker/php/setup.sh
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
#!/bin/sh
# One-shot `setup` service: installs the workbench's vendor/ inside Linux, so platform checks
# (ext-pcntl, ext-posix, ext-redis) match the app containers. The package itself is a symlink to
# the repo root (Composer path repository), so edits to src/ need no reinstall.
# (ext-pcntl, ext-posix, ext-redis) match the app containers.
set -eu
cd /repo/workbench

Expand All @@ -10,3 +9,20 @@ if [ "${1:-}" = "update" ]; then
else
composer install --no-interaction --no-progress
fi

# The package goes in as a copy, as Composer installs it from Packagist, not as the path repository's
# symlink to the repo root: PHP resolves symlinks in stack traces, and error trackers judge a frame by
# its path (App\Support\ErrorTrackers). Copied again on every run, so edits to src/ reach the stacks
# when they are recreated.
pkg=vendor/tevkert/queue-prefork
rm -rf "$pkg.copy"
mkdir "$pkg.copy"
cp -R /repo/bin /repo/config /repo/src /repo/composer.json "$pkg.copy/"

if [ -L "$pkg" ]; then
rm "$pkg"
else
rm -rf "$pkg"
fi

mv "$pkg.copy" "$pkg"
2 changes: 1 addition & 1 deletion src/Handoff.php
Original file line number Diff line number Diff line change
Expand Up @@ -421,7 +421,7 @@ private function connect(string $socket): bool
/** Starts this key's template, detached from this process's session. */
private function launch(): void
{
$template = RuntimeDir::copy($this->runtime, 'template', 'php', (string) file_get_contents(dirname(__DIR__).'/bin/template.php'));
$template = RuntimeDir::launcher($this->runtime, dirname(__DIR__));
RuntimeDir::writeAtomic($this->runtime, "{$this->key16}.ini.json", json_encode(TemplateKey::iniValues(), JSON_UNESCAPED_SLASHES));

$command = [PHP_BINARY, ...self::phpOptions()];
Expand Down
8 changes: 5 additions & 3 deletions src/Master.php
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,9 @@ final class Master

private string $statusDir = '';

/** What the template is started from (RuntimeDir::launcher). */
private string $launcher = '';

private float $lastSnapshot = 0.0;

/** @var array<string, ForkedProcess> supervisor name => current unit handle */
Expand Down Expand Up @@ -284,8 +287,7 @@ private function prepareRuntimeDir(): void
RuntimeDir::ensure($this->statusDir);

if (! self::forkedByHost()) {
$launcher = (string) file_get_contents(self::packageDir().'/bin/template.php');
RuntimeDir::writeAtomic($this->runtimeDir, 'template.php', $launcher);
$this->launcher = RuntimeDir::launcher($this->runtimeDir, self::packageDir());
}
}

Expand Down Expand Up @@ -382,7 +384,7 @@ private function templateCommand(): array
array_push($command, '-d', $setting);
}

$command[] = $this->runtimeDir.'/template.php';
$command[] = $this->launcher;

return $command;
}
Expand Down
2 changes: 1 addition & 1 deletion src/MasterHandoff.php
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ private static function exec(string $entry): void
{
$runtime = RuntimeDir::shared();
RuntimeDir::ensure($runtime);
$template = RuntimeDir::copy($runtime, 'template', 'php', (string) file_get_contents(dirname(__DIR__).'/bin/template.php'));
$template = RuntimeDir::launcher($runtime, dirname(__DIR__));
$ini = RuntimeDir::copy($runtime, 'ini', 'json', json_encode(TemplateKey::iniValues(), JSON_UNESCAPED_SLASHES));
$settings = TemplateKey::settings(dirname($entry));
$argv = Handoff::phpOptions();
Expand Down
35 changes: 35 additions & 0 deletions src/Support/RuntimeDir.php
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,41 @@ public static function writeAtomic(string $dir, string $name, string $contents):
}
}

/**
* What a template is started from: a link in $dir to the package's own bin/template.php. The command
* line keeps this private path, with nothing in it horizon:purge could match, while PHP resolves the
* link: stack traces name the launcher in the app's vendor/, so error trackers count it as library
* code, as they do artisan. Links whose release is gone are removed. A copy where no link can be made.
*/
public static function launcher(string $dir, string $packageDir): string
{
self::check($dir);
$target = $packageDir.'/bin/template.php';
$link = "{$dir}/template-".substr(hash('sha256', $target), 0, 16).'.php';

foreach (glob("{$dir}/template-*.php") ?: [] as $file) {
if ($file !== $link && is_link($file) && ! file_exists($file)) {
@unlink($file);
}
}

if (@readlink($link) === $target) {
return $link;
}

$tmp = "{$dir}/.template.".getmypid().'.'.bin2hex(random_bytes(4)).'.tmp';

if (function_exists('symlink') && @symlink($target, $tmp)) {
if (@rename($tmp, $link)) {
return $link;
}

@unlink($tmp);
}

return self::copy($dir, 'template', 'php', (string) file_get_contents($target));
}

/** A copy of a package file in $dir, named by its content, so pruning a release can't remove it. */
public static function copy(string $dir, string $name, string $extension, string $contents): string
{
Expand Down
20 changes: 18 additions & 2 deletions tests/Integration/QueueWorkStockParityTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,9 @@ public function test_a_job_sees_the_same_process_as_in_stock(): void

public function test_every_fixture_ends_as_in_stock(): void
{
// All but `crashes` (QueueWorkParityTest has it) and `calls-artisan` (its own test): random values, connections, Concurrency::run,
// batches, releases, failures, a timeout kill, a --memory exit, output.
// All but `crashes` (QueueWorkParityTest has it), `calls-artisan` and `fails-in-vendor` (their own
// tests): random values, connections, Concurrency::run, batches, releases, failures, a timeout
// kill, a --memory exit, output.
self::both(fn ($c) => self::artisan($c, 'fixtures:dispatch --queue='.self::QUEUE));
$outcomes = [];

Expand Down Expand Up @@ -75,6 +76,21 @@ public function test_a_job_keeps_its_worker_after_running_an_artisan_command():
$this->assertSame($seen[self::STOCK], $seen[self::PREFORK], 'SIGALRM does after the command what it does in stock');
}

public function test_error_trackers_see_a_failed_job_as_in_stock(): void
{
// As in StockParityTest: a job failing in vendor code only, through Flare, Sentry and Nightwatch.
self::both(fn ($c) => self::artisan($c, 'fixtures:dispatch fails-in-vendor --queue='.self::QUEUE));
$seen = [];

foreach (self::BOTH as $container) {
$seen[$container] = self::waitUntil(fn () => self::rows($container, 'ErrorTrackers'), 30, "ErrorTrackers in {$container}")[0]['value'];
}

$this->assertSame([], $seen[self::STOCK]['flare_app_frames'], 'stock: no frame is the app\'s');
$this->assertSame($seen[self::STOCK]['thrown_at'], $seen[self::STOCK]['flare_first_frame'], 'stock: Flare keeps the trace from where it was thrown');
$this->assertSame($seen[self::STOCK], $seen[self::PREFORK]);
}

public function test_concurrent_workers_draw_different_random_numbers(): void
{
// Forked workers reseed: two of them must not share a sequence (the template's seed). Each job
Expand Down
21 changes: 21 additions & 0 deletions tests/Integration/StockParityTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,27 @@ public function test_a_job_keeps_its_worker_after_running_an_artisan_command():
$this->assertSame($seen[self::STOCK], $seen[self::PREFORK], 'SIGALRM does after the command what it does in stock');
}

public function test_error_trackers_see_a_failed_job_as_in_stock(): void
{
// A job that fails in vendor code only, as a package's job does: Flare, Sentry and Nightwatch must
// see its stack trace as in stock (App\Support\ErrorTrackers). A forked worker's trace ends in the
// launcher, not in artisan, and the launcher must count as library code. The queue:work
// counterpart is in QueueWorkStockParityTest.
foreach ([self::STOCK, self::PREFORK] as $container) {
self::artisan($container, 'fixtures:dispatch fails-in-vendor');
}

$seen = [];

foreach ([self::STOCK, self::PREFORK] as $container) {
$seen[$container] = self::waitUntil(fn () => self::results($container, 'ErrorTrackers'), 30, "ErrorTrackers in {$container}")[0]['value'];
}

$this->assertSame([], $seen[self::STOCK]['flare_app_frames'], 'stock: no frame is the app\'s');
$this->assertSame($seen[self::STOCK]['thrown_at'], $seen[self::STOCK]['flare_first_frame'], 'stock: Flare keeps the trace from where it was thrown');
$this->assertSame($seen[self::STOCK], $seen[self::PREFORK]);
}

public function test_worker_output_reaches_the_horizon_log(): void
{
foreach ([self::STOCK, self::PREFORK] as $container) {
Expand Down
4 changes: 2 additions & 2 deletions tests/Mechanics/TemplateHarness.php
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,12 @@ public function __construct(array $config = [], ?int $masterPid = null, bool $ma
$master = $masterPid ?? getmypid();
$this->statusDir = $this->runtimeDir.'/'.Identity::key($master, Identity::startTime($master));
RuntimeDir::ensure($this->statusDir);
RuntimeDir::writeAtomic($this->runtimeDir, 'template.php', (string) file_get_contents($root.'/bin/template.php'));
$launcher = RuntimeDir::launcher($this->runtimeDir, $root);
$this->out = $this->runtimeDir.'/events.jsonl';
touch($this->out);

$command = [PHP_BINARY, '-d', 'opcache.enable=1', '-d', 'opcache.enable_cli=1', '-d', 'opcache.revalidate_freq=0',
'-d', 'display_errors=stderr', '-d', 'hp.pad='.str_repeat('.', 4096), '-d', 'auto_prepend_file=', $this->runtimeDir.'/template.php'];
'-d', 'display_errors=stderr', '-d', 'hp.pad='.str_repeat('.', 4096), '-d', 'auto_prepend_file=', $launcher];
$env = array_merge(getenv(), [
'HP_PACKAGE_DIR' => $root,
'HP_MASTER_PID' => (string) $master,
Expand Down
31 changes: 31 additions & 0 deletions tests/Unit/RuntimeDirTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,37 @@ public function test_refuses_a_symlink(): void
RuntimeDir::check($this->base.'/link');
}

public function test_the_launcher_is_a_link_to_the_package_file(): void
{
$dir = $this->base.'/rt';
RuntimeDir::ensure($dir);
$package = dirname(__DIR__, 2);

$launcher = RuntimeDir::launcher($dir, $package);

$this->assertStringStartsWith($dir.'/template-', $launcher, 'the command line keeps the runtime path');
$this->assertTrue(is_link($launcher));
$this->assertSame(realpath($package.'/bin/template.php'), realpath($launcher), 'PHP names the package file in stack traces');
$this->assertSame($launcher, RuntimeDir::launcher($dir, $package), 'reused');
}

public function test_launchers_of_a_removed_release_are_removed(): void
{
$dir = $this->base.'/rt';
RuntimeDir::ensure($dir);
mkdir($this->base.'/old/bin', 0700, true);
file_put_contents($this->base.'/old/bin/template.php', '<?php');
$old = RuntimeDir::launcher($dir, $this->base.'/old');
$copy = RuntimeDir::copy($dir, 'template', 'php', '<?php // 1.1.1');
exec('rm -rf '.escapeshellarg($this->base.'/old'));

$current = RuntimeDir::launcher($dir, dirname(__DIR__, 2));

$this->assertFalse(is_link($old), 'the dangling link is gone');
$this->assertFileExists($copy, 'a copy from an earlier version stays: a template may be starting from it');
$this->assertTrue(is_link($current));
}

public function test_never_reads_a_symlinked_status_file(): void
{
$dir = $this->base.'/rt';
Expand Down
10 changes: 7 additions & 3 deletions workbench/app/Console/Commands/FixturesDispatch.php
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,13 @@
use App\Jobs\Succeeds;
use App\Jobs\TimesOut;
use Illuminate\Console\Command;
use Illuminate\Foundation\Console\QueuedCommand;

/** Dispatches fixture jobs by name. With no names, every fixture except `crashes` and `calls-artisan`. */
/** Dispatches fixture jobs by name. With no names, every fixture except `crashes`, `calls-artisan` and `fails-in-vendor`. */
class FixturesDispatch extends Command
{
protected $signature = 'fixtures:dispatch
{fixtures?* : fixture names (see --list); default: all but crashes and calls-artisan}
{fixtures?* : fixture names (see --list); default: all but crashes, calls-artisan and fails-in-vendor}
{--queue=default : queue to dispatch on}
{--times=1 : how many of each}
{--sleep=3 : seconds for sleeps}
Expand All @@ -46,7 +47,7 @@ public function handle(): int
return self::SUCCESS;
}

$names = $this->argument('fixtures') ?: array_values(array_diff(array_keys($fixtures), ['crashes', 'calls-artisan']));
$names = $this->argument('fixtures') ?: array_values(array_diff(array_keys($fixtures), ['crashes', 'calls-artisan', 'fails-in-vendor']));
$unknown = array_diff($names, array_keys($fixtures));

if ($unknown) {
Expand Down Expand Up @@ -87,6 +88,9 @@ private function fixtures(): array
'runs-concurrency' => fn () => new RunsConcurrency,
'dispatches-batch' => fn () => new DispatchesBatch,
'calls-artisan' => fn () => new CallsArtisan((int) $this->option('work')),
// Laravel's own job, failing in Symfony's code: a trace without any of the app's frames, as for
// a package's job. App\Support\ErrorTrackers records what error trackers make of it.
'fails-in-vendor' => fn () => new QueuedCommand(['fixtures:no-such-command', []]),
];
}
}
47 changes: 47 additions & 0 deletions workbench/app/Support/ErrorTrackers.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
<?php

namespace App\Support;

use Laravel\Nightwatch\Location;
use Sentry\Options;
use Sentry\Serializer\RepresentationSerializer;
use Sentry\StacktraceBuilder;
use Spatie\FlareClient\Flare;
use Spatie\FlareClient\FlareConfig;
use Throwable;

/**
* What three error trackers make of an exception's stack trace, through their own code, set up as their
* Laravel integrations set them up, with nothing sent. Each decides by path which frames are the app's
* own code, and a forked worker's trace ends in the launcher it was forked from, not in `artisan`:
*
* - Flare drops every frame above the first one it counts as the app's (ReportFactory::buildStacktrace);
* - Sentry marks frames in-app (sentry-laravel's in_app_exclude) and groups by them;
* - Nightwatch places an exception at its first frame outside vendor/ (Location::forException).
*/
class ErrorTrackers
{
private static ?Flare $flare = null;

public static function record(Throwable $e): void
{
$at = fn (string $file, ?int $line): string => str_replace(base_path().'/', '', $file).':'.$line;

self::$flare ??= Flare::make(FlareConfig::make('workbench')->useDefaults()->applicationPath(base_path()));
$flare = self::$flare->createReport($e)->toArray()['stacktrace'];

// sentry-laravel's defaults (ServiceProvider::configureAndRegisterClient)
$options = new Options(['prefixes' => [base_path()], 'in_app_exclude' => [base_path('vendor'), base_path('artisan')]]);
$sentry = (new StacktraceBuilder($options, new RepresentationSerializer($options)))->buildFromException($e)->getFrames();

[$file, $line] = (new Location(base_path(), public_path()))->forException($e);

JobResult::record('ErrorTrackers', [
'thrown_at' => $at($e->getFile(), $e->getLine()),
'flare_first_frame' => $at($flare[0]['file'], $flare[0]['lineNumber']),
'flare_app_frames' => array_map(fn (array $frame) => $at($frame['file'], $frame['lineNumber']), array_values(array_filter($flare, fn (array $frame) => $frame['isApplicationFrame']))),
'sentry_in_app' => array_map(fn ($frame) => $at($frame->getAbsoluteFilePath(), $frame->getLine()), array_values(array_filter($sentry, fn ($frame) => $frame->isInApp()))),
'nightwatch_location' => $at(str_starts_with($file, '/') ? $file : base_path($file), $line),
]);
}
}
8 changes: 8 additions & 0 deletions workbench/bootstrap/app.php
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
use Illuminate\Foundation\Configuration\Exceptions;
use Illuminate\Foundation\Configuration\Middleware;
use Illuminate\Http\Request;
use Symfony\Component\Console\Exception\CommandNotFoundException;

return Application::configure(basePath: dirname(__DIR__))
->withRouting(
Expand All @@ -18,4 +19,11 @@
$exceptions->shouldRenderJsonWhen(
fn (Request $request) => $request->is('api/*') || $request->expectsJson(),
);

// The fails-in-vendor fixture: what error trackers would report (App\Support\ErrorTrackers).
$exceptions->report(function (CommandNotFoundException $e): void {
if (str_contains($e->getMessage(), 'fixtures:no-such-command')) {
App\Support\ErrorTrackers::record($e);
}
});
})->create();
2 changes: 2 additions & 0 deletions workbench/composer.json
Original file line number Diff line number Diff line change
Expand Up @@ -55,9 +55,11 @@
"opcodesio/log-viewer": "^3.24",
"platformcommunity/flysystem-bunnycdn": "^3.3",
"ramsey/uuid": "^4.9",
"sentry/sentry": "^4.38",
"socialiteproviders/discord": "^4.2",
"socialiteproviders/manager": "^4.9",
"socialiteproviders/twitch": "^5.4",
"spatie/flare-client-php": "^3.6",
"spatie/fork": "^1.2",
"spatie/laravel-activitylog": "^4.12",
"spatie/laravel-health": "^1.40",
Expand Down
Loading
Loading