Runner reliability: waitForPageLoaded, exit codes, env, bin, --suites

README.md

@@ -28,6 +28,29 @@ Resolution priority: fluent `onVersion()` → `$psVersion` property → `PRESTAF
 
 `onVersion()` throws `InvalidArgumentException` on malformed input (expected format: `1.7`, `1.7.8`, `1.7.8.11`, `9`, `9.0`, `9.0.1`, etc.).
 
+## Run only some suite folders
+
+`prestaflow run <path>` runs every suite under `<path>`. To run only some of
+its sub-folders, list them, comma-separated, relative to `<path>`:
+
+```bash
+./vendor/bin/prestaflow run tests --suites=BackOffice,FrontOffice/Checkout
+PRESTAFLOW_SUITES=BackOffice,FrontOffice/Checkout ./vendor/bin/prestaflow run tests
+```
+
+- Each name is a sub-folder of `<path>`, scanned recursively; nested paths such
+  as `FrontOffice/Checkout` are allowed. The run takes the union of the folders.
+- `--suites` wins over `PRESTAFLOW_SUITES`. The GitHub Action sets
+  `PRESTAFLOW_SUITES` from its `suites` input.
+- A name that matches no folder fails the run (non-zero exit code), listing the
+  missing names and the available sub-folders. A typo never becomes a green
+  job that ran zero tests.
+- A filtered run that ends up with no suite at all (an empty folder, or
+  nothing left after `--group` / `--draft`) fails too, for the same reason.
+- Absolute paths and `..` are refused: the filter can only narrow `<path>`.
+- `--group` and `--draft` still apply, on the suites of the selected folders.
+- Unset or empty: every suite under `<path>` runs, as before.
+
 ## Run a suite against a throwaway shop
 
 `docker-compose.yml` boots a disposable PrestaShop from the official

bin/prestaflow

@@ -28,48 +28,6 @@ if (!$autoloadLoaded) {
     exit(1);
 }
 
-use PrestaFlow\Library\Command\ExecuteSuite;
-use Symfony\Component\Console\Application;
-use Symfony\Component\Console\Command\Command;
-use Symfony\Component\Console\ConsoleEvents;
-use Symfony\Component\Console\Event\ConsoleErrorEvent;
-use Symfony\Component\Console\Style\SymfonyStyle;
-use Symfony\Component\EventDispatcher\EventDispatcher;
-
-$dispatcher = new EventDispatcher();
-
-$dispatcher->addListener(ConsoleEvents::ERROR, function (ConsoleErrorEvent $event): void {
-
-    $input = $event->getInput();
-    $output = $event->getOutput();
-
-    $io = new SymfonyStyle($input, $output);
-
-    $command = $event->getCommand();
-
-    $error = $event->getError();
-
-    $io->newLine();
-
-    $io->writeln(sprintf('<fg=red;options=bold>ERROR</> <fg=white>%s</>', $error->getMessage()));
-    $io->writeln(sprintf('<fg=gray;options=bold>TRACE</> <fg=white>%s</>', $error->getFile() . ':' . $error->getLine()));
-
-    foreach ($error->getTrace() as $trace) {
-        $io->writeln(sprintf('<fg=gray;options=bold>TRACE</> <fg=white>%s</>', ($trace['file'] ?? '[internal]') . ':' . ($trace['line'] ?? '?')));
-    }
-
-    // gets the current exit code (the exception code)
-    $exitCode = $event->getExitCode();
-
-    $event->setExitCode(Command::SUCCESS);
-
-    // changes the exception to another one
-    $event->setError(new \LogicException('Caught exception', $exitCode, $event->getError()));
-});
-
-$application = new Application();
-
-$application->add(new ExecuteSuite());
-$application->setDispatcher($dispatcher);
-
-$application->run();
+// Toute la configuration vit dans PrestaFlow\Library\Console\Application (testée) :
+// une exception non rattrapée y donne un code de sortie non nul.
+(new PrestaFlow\Library\Console\Application())->run();

composer.json

@@ -3,6 +3,9 @@
     "description": "PrestaFlow is an open-source set of prebuilt tests components, ready-to-use examples made for the PrestaShop E-commerce software.",
     "type": "library",
     "license": "MIT",
+    "bin": [
+        "bin/prestaflow"
+    ],
     "autoload": {
         "psr-4": {
             "PrestaFlow\\Library\\": "src/",

src/Command/ExecuteSuite.php

@@ -55,6 +55,7 @@ protected function configure(): void
             ->addOption('visual-report', null, InputOption::VALUE_OPTIONAL, 'Écrit un rapport de régression visuelle (HTML)', false)
             ->addOption('visual-report-tz', null, InputOption::VALUE_REQUIRED, 'Fuseau horaire du stamp du rapport visuel (ex. Europe/Brussels). Défaut : env PRESTAFLOW_TZ ou UTC.')
             ->addOption('draft', 'd', InputOption::VALUE_NEGATABLE, 'Draft mode')
+            ->addOption('suites', null, InputOption::VALUE_REQUIRED, 'Comma-separated sub-folders of the suites path to run (e.g. BackOffice,FrontOffice/Checkout). Overrides env PRESTAFLOW_SUITES.')
             ->addArgument('folder', InputArgument::OPTIONAL, 'The folder name', 'tests')
             ->addOption(
                 'group',
@@ -138,7 +139,11 @@ public function execute(InputInterface $input, OutputInterface $output): int
         $junitPath = ($junitOption === false) ? null : ($junitOption ?: 'prestaflow/junit.xml');
 
         try {
-            $testSuites = $this->resolveSuitePaths((string) $input->getArgument('folder'));
+            // --suites wins over PRESTAFLOW_SUITES (set by the GitHub Action).
+            $suitesFilter = $this->parseSuitesFilter(
+                $input->getOption('suites') ?? Env::get('PRESTAFLOW_SUITES')
+            );
+            $testSuites = $this->resolveSuitePaths((string) $input->getArgument('folder'), $suitesFilter);
         } catch (Error $e) {
             $this->sections['progressIndicator']->finish('Finished');
             $this->sections['progressBar']->clear();
@@ -149,6 +154,7 @@ public function execute(InputInterface $input, OutputInterface $output): int
             $this->sections['progressIndicator']->finish('Finished');
             $this->sections['progressBar']->clear();
 
+            $this->failOnEmptyFilteredRun($suitesFilter, (string) $input->getArgument('folder'));
             $this->success('Tests folder is empty', newLine: true);
             return Command::SUCCESS;
         };
@@ -255,6 +261,7 @@ public function execute(InputInterface $input, OutputInterface $output): int
         }
 
         if (!$nbSuites) {
+            $this->failOnEmptyFilteredRun($suitesFilter, (string) $input->getArgument('folder'));
             $this->success('Tests folder is empty', newLine: true);
             return Command::SUCCESS;
         };
@@ -443,14 +450,24 @@ public function candidatePaths(string $argument): array
      *
      * @throws Error when the argument matches neither a directory nor a suite file
      */
-    public function resolveSuitePaths(string $argument): array
+    public function resolveSuitePaths(string $argument, array $subFolders = []): array
     {
         foreach ($this->candidatePaths($argument) as $path) {
             if (is_dir($path)) {
-                return $this->getTestsSuites($path);
+                return $subFolders === []
+                    ? $this->getTestsSuites($path)
+                    : $this->getTestsSuitesIn($path, $subFolders);
             }
 
             if (is_file($path)) {
+                if ($subFolders !== []) {
+                    throw new Error(sprintf(
+                        '[%s] is a single suite file: a suites filter (%s) needs a folder',
+                        $path,
+                        implode(', ', $subFolders)
+                    ));
+                }
+
                 if (!str_ends_with($path, '.php')) {
                     throw new Error(sprintf('[%s] is not a PHP suite file', $path));
                 }
@@ -462,6 +479,97 @@ public function resolveSuitePaths(string $argument): array
         throw new Error(sprintf('The suites path [%s] doesn\'t seem to exist', $argument));
     }
 
+    /**
+     * Parse a suites filter (--suites or PRESTAFLOW_SUITES): comma-separated
+     * sub-folders of the suites path, nested ones allowed (FrontOffice/Checkout).
+     * Names are trimmed, empty ones dropped, duplicates removed.
+     *
+     * @return array<int, string> an empty array means "no filter"
+     *
+     * @throws Error on an absolute path or a `..` segment: the filter may only
+     *               narrow the suites path, never leave it
+     */
+    public function parseSuitesFilter(?string $raw): array
+    {
+        $names = [];
+        foreach (explode(',', (string) $raw) as $name) {
+            $name = trim(str_replace('\\', '/', $name));
+            if ($name === '') {
+                continue;
+            }
+
+            if (str_starts_with($name, '/') || preg_match('#^[A-Za-z]:#', $name)) {
+                throw new Error(sprintf('Suites filter [%s]: absolute paths are not allowed, name a sub-folder of the suites path', $name));
+            }
+
+            $segments = array_values(array_filter(explode('/', $name), fn ($segment) => $segment !== '' && $segment !== '.'));
+            if (in_array('..', $segments, true)) {
+                throw new Error(sprintf('Suites filter [%s]: ".." is not allowed, name a sub-folder of the suites path', $name));
+            }
+            if ($segments === []) {
+                continue;
+            }
+
+            $names[] = implode('/', $segments);
+        }
+
+        return array_values(array_unique($names));
+    }
+
+    /**
+     * A run narrowed by --suites / PRESTAFLOW_SUITES that ends up with no suite
+     * is a mistake in the filter, not an empty project: fail it rather than
+     * report a green run of zero tests. Unfiltered, an empty folder still passes.
+     *
+     * @throws Error when $suitesFilter is not empty
+     */
+    protected function failOnEmptyFilteredRun(array $suitesFilter, string $root): void
+    {
+        if ($suitesFilter !== []) {
+            throw new Error(sprintf(
+                'Suites filter [%s] selected no suite under [%s]',
+                implode(', ', $suitesFilter),
+                $root
+            ));
+        }
+    }
+
+    /**
+     * Suites of the given sub-folders of $root (the union, without duplicates).
+     *
+     * @return array<int, string>
+     *
+     * @throws Error when a name matches no folder, so that a filter typo never
+     *               turns into a green run of zero tests
+     */
+    public function getTestsSuitesIn(string $root, array $subFolders): array
+    {
+        $missing = array_values(array_filter($subFolders, fn ($name) => !is_dir($root . '/' . $name)));
+        if ($missing !== []) {
+            $available = array_values(array_filter(
+                scandir($root) ?: [],
+                fn ($entry) => $entry !== '.' && $entry !== '..' && is_dir($root . '/' . $entry)
+            ));
+            sort($available);
+
+            throw new Error(sprintf(
+                'Suites filter: no folder [%s] under [%s]. Available sub-folders: %s',
+                implode(', ', $missing),
+                $root,
+                $available === [] ? '(none)' : implode(', ', $available)
+            ));
+        }
+
+        $testSuites = [];
+        foreach ($subFolders as $name) {
+            foreach ($this->getTestsSuites($root . '/' . $name) as $suitePath) {
+                $testSuites[] = $suitePath;
+            }
+        }
+
+        return array_values(array_unique($testSuites));
+    }
+
     public function getTestsSuites($folderPath)
     {
         $testSuites = [];

src/Console/Application.php

@@ -0,0 +1,40 @@
+<?php
+
+namespace PrestaFlow\Library\Console;
+
+use PrestaFlow\Library\Command\ExecuteSuite;
+use Symfony\Component\Console\Application as BaseApplication;
+use Symfony\Component\Console\Output\OutputInterface;
+
+/**
+ * The `prestaflow` console application.
+ *
+ * Lives here rather than in bin/prestaflow so its exit codes can be tested.
+ * An uncaught exception or error (Chrome that does not start, a suites path
+ * that does not exist) is rendered as ERROR/TRACE lines and ends the run with
+ * a non-zero exit code; a run whose assertions fail still exits with 1, and a
+ * clean run with 0.
+ */
+class Application extends BaseApplication
+{
+    public function __construct()
+    {
+        parent::__construct('PrestaFlow', \PrestaFlow\Library\Traits\AppVersion::APP_VERSION);
+
+        $this->add(new ExecuteSuite());
+
+        // Most failures inside a run are \Error (resolveSuitePaths() throws one):
+        // without this they would escape run() as a PHP fatal error.
+        $this->setCatchErrors(true);
+    }
+
+    protected function doRenderThrowable(\Throwable $e, OutputInterface $output): void
+    {
+        $output->writeln(sprintf('<fg=red;options=bold>ERROR</> <fg=white>%s</>', $e->getMessage()));
+        $output->writeln(sprintf('<fg=gray;options=bold>TRACE</> <fg=white>%s</>', $e->getFile() . ':' . $e->getLine()));
+
+        foreach ($e->getTrace() as $trace) {
+            $output->writeln(sprintf('<fg=gray;options=bold>TRACE</> <fg=white>%s</>', ($trace['file'] ?? '[internal]') . ':' . ($trace['line'] ?? '?')));
+        }
+    }
+}

src/Pages/CommonPage.php

@@ -125,8 +125,23 @@ public function __call($name, $arguments)
         $page = $this->getPage();
 
         if (!is_null($page) && method_exists($page, $name)) {
-            call_user_func_array([$page, $name], $arguments);
+            return call_user_func_array([$page, $name], $arguments);
         }
+
+        // No browser page yet: keep the historical no-op for real chrome-php
+        // methods, but never for a name nobody defines.
+        if (is_null($page) && method_exists(DomPage::class, $name)) {
+            return null;
+        }
+
+        // A typo or a method that lives elsewhere (e.g. waitForNavigation(), which
+        // belongs to PageNavigation, not Page) used to be swallowed silently, so
+        // the step "passed" without doing anything.
+        throw new \BadMethodCallException(sprintf(
+            'Call to undefined method %s::%s() (not defined on the page object nor on the chrome-php page).',
+            static::class,
+            $name
+        ));
     }
 
     public function setGlobals($globals)
@@ -546,9 +561,27 @@ public function goToUrl(string $url)
         $this->getPage()->navigate($url)->waitForNavigation(DomPage::DOM_CONTENT_LOADED);
     }
 
-    public function waitForPageLoaded()
+    /**
+     * Wait until the current document has been parsed (DOMContentLoaded, i.e.
+     * readyState "interactive" or "complete") and has a <body>.
+     *
+     * This polls the page state; it does not observe a navigation. Right after
+     * an action that triggers one, the previous document may still report
+     * itself as ready, so prefer waiting on navigate()->waitForNavigation() or
+     * on an element of the next page when you have one.
+     *
+     * @throws TimeoutException when the page is still loading after $timeout ms
+     */
+    public function waitForPageLoaded(int $timeout = 30000): void
     {
-        $this->waitForNavigation(DomPage::DOM_CONTENT_LOADED, 10000);
+        $loaded = $this->waitForJsCondition(
+            "document.readyState !== 'loading' && !!document.body",
+            $timeout
+        );
+
+        if (!$loaded) {
+            throw new TimeoutException(sprintf('Page did not finish loading within %d ms.', $timeout));
+        }
     }
 
     public function getTextContent($selector, $index = 1, $waitForSelector = true, $timeout = 3000)

src/Pages/FrontOfficePage.php

@@ -75,12 +75,12 @@ public function goToPage($page = null, $params = null)
             if ($hasDebugMoved) {
                 Expect::setWarning('debug-mode');
                 $this->click('a');
-                $this->waitForNavigation();
+                $this->waitForPageLoaded();
             }
         } catch (OperationTimedOut | Exception $e) {
             Expect::setWarning('debug-mode');
             $this->click('a');
-            $this->waitForNavigation();
+            $this->waitForPageLoaded();
         }
     }
 

src/Pages/v9/FrontOffice/Product/Page.php

@@ -33,7 +33,7 @@ public function goToProduct(int $productId = 0)
     {
         $this->goToPage('product', $productId);
 
-        $this->waitForNavigation();
+        $this->waitForPageLoaded();
     }
 
     /**

src/Tests/TestsSuite.php

@@ -635,7 +635,7 @@ protected function presetBasicAuth(): void
      */
     protected function presetExtraHeadersFromEnv(): void
     {
-        $raw = $_ENV['PRESTAFLOW_EXTRA_HEADERS'] ?? null;
+        $raw = Env::get('PRESTAFLOW_EXTRA_HEADERS');
         if ($raw === null || $raw === '') {
             return;
         }