diff --git a/.env.default b/.env.default index df58ae0..69d76ce 100644 --- a/.env.default +++ b/.env.default @@ -11,6 +11,10 @@ # $ source .env ### +# Optional label to distinguish this environment in test reports, for example +# "shared", "vps", or "cloud". Use a single alphanumeric keyword. +export WPT_LABEL="" + # Path to the directory where files can be prepared before being delivered to the environment. export WPT_PREPARE_DIR="/tmp/wp-test-runner" @@ -35,7 +39,18 @@ export WPT_DB_HOST="" # (Optionally) set a custom table prefix to permit concurrency against the same database. export WPT_TABLE_PREFIX="${WPT_TABLE_PREFIX-wptests_}" -# (Optionally) define the PHP executable to be called +# PHP executable to be called. Default: php +# +# A single binary: +# export WPT_PHP_EXECUTABLE="php" +# export WPT_PHP_EXECUTABLE="/usr/bin/php8.1" +# +# Multiple versions (version=path, separated by semicolons). Each version +# gets its own prepare/test directories and database table prefix: +# export WPT_PHP_EXECUTABLE="8.1=/bin/php8.1;8.2=/bin/php8.2;8.3=/bin/php8.3" +# +# Use as many versions as you'd like, but keep in mind that it will take more time. +# Ideally all versions offered to users are tested. export WPT_PHP_EXECUTABLE="${WPT_PHP_EXECUTABLE-php}" # (Optionally) define the PHPUnit command execution call. @@ -83,3 +98,12 @@ export WPT_FLAVOR=0 # 2 = ms-files # 3 = external-http export WPT_EXTRATESTS=0 + +# Whether to queue recent wordpress-develop commits instead of only the latest. +# +# 0 = Off. Test only the most recent commit when the runner starts. +# 1 = On. Query the last 30 commits and queue any that have not been tested yet. +# +# Regardless of this value, commits.json tracks SHAs that were already tested +# and reported so the suite is not re-run for the same commit. +export WPT_COMMITS=0 diff --git a/.gitignore b/.gitignore index 023e69a..7554acc 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ vendor/ .cache/ package-lock.json commit.json +commits.json ignore.json # Exclude the default test directory. diff --git a/README.md b/README.md index 73917db..d7432d2 100644 --- a/README.md +++ b/README.md @@ -626,6 +626,28 @@ journalctl -u testrunner.timer journalctl -n 120 -u testrunner.service ``` +## Multiple PHP versions, environments, and commits + +These options are configured in `.env`. Broader documentation updates live in a separate pull request; this section only covers the runner behavior added for multi-PHP, environment labels, and commit tracking. + +**Environment label.** Set `WPT_LABEL` to a short alphanumeric keyword such as `shared`, `vps`, or `cloud`. The label is included in the reported environment details so results from different setups on the same host can be distinguished. + +**Multiple PHP versions.** `WPT_PHP_EXECUTABLE` still defaults to `php`. A single binary path continues to work as before. To test more than one version, use `version=path` entries separated by semicolons: + +```bash +export WPT_PHP_EXECUTABLE="8.1=/bin/php8.1;8.2=/bin/php8.2;8.3=/bin/php8.3" +``` + +Each version uses its own prepare/test directory (the version is appended in plain text, for example `/tmp/wp-test-runner-8-1`) and a unique database table prefix so runs do not collide. + +**Commit tracking.** The runner writes `commits.json` (gitignored; copied from `commits.json.example` when missing): + +- `executed_commits`: SHAs that were successfully tested and reported +- `pending_commits`: SHAs waiting to be tested (oldest first) +- `testing_commit`: the SHA currently being tested (at most one) + +`WPT_COMMITS=0` tests only the latest commit when the runner starts. `WPT_COMMITS=1` queries the last 30 wordpress-develop commits (the GitHub API default page size) and queues any that have not been tested yet. In both modes, a commit already present in `commits.json` is skipped. + ## Contributing If you have questions about the process or run into test failures along the way, please [open an issue in the project repository](https://github.com/WordPress/phpunit-test-runner/issues) and we’ll help diagnose/get the documentation updated. Alternatively, you can also pop into the `#hosting` channel on [WordPress.org Slack](https://make.wordpress.org/chat/) for help. diff --git a/cleanup.php b/cleanup.php index dddc5fb..2669e27 100644 --- a/cleanup.php +++ b/cleanup.php @@ -1,22 +1,15 @@ '' !== $test_dir ? $test_dir : '/tmp/wp-test-runner', @@ -86,94 +92,503 @@ function setup_runner_env_vars() { 'WPT_PREPARE_DIR' => '' !== $prepare_dir ? $prepare_dir : '/tmp/wp-test-runner', 'WPT_SSH_CONNECT' => trim( getenv( 'WPT_SSH_CONNECT' ) ), 'WPT_SSH_OPTIONS' => '' !== $ssh_options ? $ssh_options : '-o StrictHostKeyChecking=no', - 'WPT_PHP_EXECUTABLE' => '' !== $php_exec ? $php_exec : 'php', + 'WPT_PHP_EXECUTABLE' => $php_executables[0]['bin'], + 'WPT_PHP_EXECUTABLES' => $php_executables, 'WPT_RM_TEST_DIR_CMD' => '' !== $rm_test_dir ? $rm_test_dir : 'rm -r ' . $runner_configuration['WPT_TEST_DIR'], 'WPT_REPORT_API_KEY' => trim( getenv( 'WPT_REPORT_API_KEY' ) ), 'WPT_DEBUG' => (bool) getenv( 'WPT_DEBUG' ), + 'WPT_LABEL' => $label, + 'WPT_COMMITS' => ( '1' === $commits || 'true' === $commits || 'on' === $commits ), + ) + ); +} + +/** + * Converts a PHP version string into a filesystem-safe directory suffix. + * + * @param string $version PHP version such as '8.1' or 'default'. + * @return string Version with dots replaced by hyphens. + */ +function directory_name_from_php_version( $version ) { + return str_replace( '.', '-', (string) $version ); +} + +/** + * Parses WPT_PHP_EXECUTABLE into one or more PHP binaries. + * + * A single value such as `php` or `/usr/bin/php8.1` is treated as one executable. + * Values containing `=` and/or `;` are treated as a version=path map, for example + * `8.1=/bin/php8.1;8.2=/bin/php8.2`. + * + * @param string $php_executable Raw WPT_PHP_EXECUTABLE value. + * @return array[] { + * @type array ...$0 { + * @type string $version Version label (`default` for a single binary). + * @type string $bin Path or command used to invoke PHP. + * @type string $suffix Directory suffix; empty for a single binary. + * } + * } + */ +function parse_php_executables( $php_executable ) { + $php_executable = trim( $php_executable ); + if ( '' === $php_executable ) { + $php_executable = 'php'; + } + + $is_multi = ( false !== strpos( $php_executable, '=' ) || false !== strpos( $php_executable, ';' ) ); + + if ( ! $is_multi ) { + return array( + array( + 'version' => 'default', + 'bin' => $php_executable, + 'suffix' => '', + ), + ); + } + + $executables = array(); + $entries = explode( ';', $php_executable ); + foreach ( $entries as $entry ) { + $entry = trim( $entry ); + if ( '' === $entry ) { + continue; + } + + $parts = array_map( 'trim', explode( '=', $entry, 2 ) ); + if ( 2 !== count( $parts ) || '' === $parts[0] || '' === $parts[1] ) { + continue; + } + + $executables[] = array( + 'version' => $parts[0], + 'bin' => $parts[1], + 'suffix' => '-' . directory_name_from_php_version( $parts[0] ), + ); + } + + if ( empty( $executables ) ) { + return array( + array( + 'version' => 'default', + 'bin' => 'php', + 'suffix' => '', + ), + ); + } + + return $executables; +} + +/** + * Returns prepare/test directory paths for a parsed PHP executable. + * + * @param array $runner_vars Configuration from setup_runner_env_vars(). + * @param array $php One item from parse_php_executables(). + * @return array { + * @type string $prepare_dir Prepare directory for this PHP version. + * @type string $test_dir Test directory for this PHP version. + * @type string $rm_cmd Command used to remove the test directory. + * } + */ +function get_php_run_paths( $runner_vars, $php ) { + $prepare_dir = $runner_vars['WPT_PREPARE_DIR'] . $php['suffix']; + $test_dir = $runner_vars['WPT_TEST_DIR'] . $php['suffix']; + $custom_rm = trim( getenv( 'WPT_RM_TEST_DIR_CMD' ) ); + + if ( '' !== $custom_rm && '' === $php['suffix'] ) { + $rm_cmd = $custom_rm; + } else { + $rm_cmd = 'rm -r ' . $test_dir; + } + + return array( + 'prepare_dir' => $prepare_dir, + 'test_dir' => $test_dir, + 'rm_cmd' => $rm_cmd, + ); +} + +/** + * Whether any versioned prepare directory already exists. + * + * @param array $runner_vars Configuration from setup_runner_env_vars(). + * @return bool + */ +function any_prepare_directory_exists( $runner_vars ) { + foreach ( $runner_vars['WPT_PHP_EXECUTABLES'] as $php ) { + $paths = get_php_run_paths( $runner_vars, $php ); + if ( is_dir( $paths['prepare_dir'] ) ) { + return true; + } + } + + return false; +} + +/** + * Exits successfully when prepare did not create an environment (nothing to test). + * + * @param array $runner_vars Configuration from setup_runner_env_vars(). + * @return void + */ +function skip_if_no_prepared_environment( $runner_vars ) { + if ( any_prepare_directory_exists( $runner_vars ) ) { + return; + } + + log_message( 'No prepared test environment found. Skipping.' ); + exit( 0 ); +} + +/** + * Absolute path to the runner's commits.json state file. + * + * @return string + */ +function runner_commits_file_path() { + return __DIR__ . '/commits.json'; +} + +/** + * Default commits.json structure. + * + * executed_commits: SHAs that were successfully tested and reported. + * pending_commits: SHAs queued to be tested, oldest first. + * testing_commit: SHA currently being tested (empty string if none). Never more than one. + * + * @return array + */ +function default_commits_state() { + return array( + 'executed_commits' => array(), + 'pending_commits' => array(), + 'testing_commit' => '', + ); +} + +/** + * Creates commits.json from the example file when it is missing. + * + * @return void + */ +function ensure_commits_file() { + $file = runner_commits_file_path(); + $example = __DIR__ . '/commits.json.example'; + + if ( file_exists( $file ) ) { + return; + } + + if ( file_exists( $example ) ) { + copy( $example, $file ); + return; + } + + save_commits_state( default_commits_state() ); +} + +/** + * Loads and normalizes commits.json. + * + * @return array + */ +function load_commits_state() { + ensure_commits_file(); + + $decoded = json_decode( (string) file_get_contents( runner_commits_file_path() ), true ); + $state = default_commits_state(); + + if ( ! is_array( $decoded ) ) { + return $state; + } + + if ( ! empty( $decoded['executed_commits'] ) && is_array( $decoded['executed_commits'] ) ) { + $state['executed_commits'] = array_values( array_filter( $decoded['executed_commits'] ) ); + } + + if ( ! empty( $decoded['pending_commits'] ) && is_array( $decoded['pending_commits'] ) ) { + $state['pending_commits'] = array_values( array_filter( $decoded['pending_commits'] ) ); + } + + if ( isset( $decoded['testing_commit'] ) ) { + if ( is_array( $decoded['testing_commit'] ) ) { + $state['testing_commit'] = ! empty( $decoded['testing_commit'][0] ) ? (string) $decoded['testing_commit'][0] : ''; + } else { + $state['testing_commit'] = (string) $decoded['testing_commit']; + } + } + + return $state; +} + +/** + * Writes commits.json. + * + * @param array $state Commit tracking state. + * @return void + */ +function save_commits_state( $state ) { + $state = array_merge( default_commits_state(), $state ); + file_put_contents( + runner_commits_file_path(), + json_encode( $state, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES ) . "\n" + ); +} + +/** + * SHAs already known to the runner (executed, pending, or in progress). + * + * @param array $state Commit tracking state. + * @return string[] + */ +function known_commit_shas( $state ) { + $testing = '' !== $state['testing_commit'] ? array( $state['testing_commit'] ) : array(); + + return array_values( + array_unique( + array_merge( $state['executed_commits'], $state['pending_commits'], $testing ) ) ); } /** - * Executes a set of shell commands. + * Fetches a URL body using cURL when available, otherwise file_get_contents(). * - * Each command is logged before being executed. + * @param string $url URL to fetch. + * @return string|false Response body, or false on failure. + */ +function fetch_url_contents( $url ) { + if ( function_exists( 'curl_init' ) ) { + $process = curl_init( $url ); + curl_setopt( $process, CURLOPT_TIMEOUT, 30 ); + curl_setopt( $process, CURLOPT_RETURNTRANSFER, true ); + curl_setopt( $process, CURLOPT_USERAGENT, 'WordPress PHPUnit Test Runner' ); + curl_setopt( + $process, + CURLOPT_HTTPHEADER, + array( + 'Accept: application/vnd.github+json', + ) + ); + + $response = curl_exec( $process ); + $status_code = (int) curl_getinfo( $process, CURLINFO_HTTP_CODE ); + curl_close( $process ); + + if ( 200 !== $status_code || false === $response ) { + return false; + } + + return $response; + } + + $context = stream_context_create( + array( + 'http' => array( + 'method' => 'GET', + 'header' => "User-Agent: WordPress PHPUnit Test Runner\r\nAccept: application/vnd.github+json\r\n", + 'timeout' => 30, + ), + ) + ); + + return file_get_contents( $url, false, $context ); +} + +/** + * Fetches recent wordpress-develop commit SHAs from the GitHub API (newest first). * - * When a non-zero return code is encountered, the error message is displayed - * and the runner will fail. + * The GitHub API defaults to 30 results per page and caps at 100. * - * @param array $operations A list of shell commands (strings) to execute. - * Each command should be a valid shell command and properly escaped for safety. - * The commands are executed in the order they appear in the array. + * @param int $per_page Number of commits to request. Default 30. + * @return string[] + */ +function fetch_wordpress_develop_commit_shas( $per_page = 30 ) { + $per_page = (int) $per_page; + if ( $per_page < 1 ) { + $per_page = 1; + } elseif ( $per_page > 100 ) { + $per_page = 100; + } + + $url = 'https://api.github.com/repos/WordPress/wordpress-develop/commits?per_page=' . $per_page; + $response = fetch_url_contents( $url ); + + if ( false === $response ) { + log_message( 'Warning: Could not fetch wordpress-develop commits from GitHub API.' ); + return array(); + } + + $decoded = json_decode( $response, true ); + if ( ! is_array( $decoded ) ) { + return array(); + } + + $shas = array(); + foreach ( $decoded as $commit ) { + if ( ! empty( $commit['sha'] ) ) { + $shas[] = $commit['sha']; + } + } + + return $shas; +} + +/** + * Resolves the current wordpress-develop HEAD SHA via git ls-remote. * - * @return void This function does not return a value. However, it will output - * the result of each shell command to the standard output and log the - * execution. It will also halt on the first command that fails, displaying an - * error message. + * @return string SHA or an empty string on failure. + */ +function fetch_wordpress_develop_head_sha() { + $output = array(); + $retval = 0; + exec( 'git ls-remote https://github.com/WordPress/wordpress-develop.git HEAD', $output, $retval ); + if ( 0 !== $retval || empty( $output[0] ) ) { + return ''; + } + + $parts = preg_split( '/\s+/', trim( $output[0] ) ); + return isset( $parts[0] ) ? $parts[0] : ''; +} + +/** + * Queues untested wordpress-develop commits and returns the SHA to test next. * - * @uses log_message() to log each operation before execution. This can be used - * for debugging or auditing purposes. + * When $fetch_history is true, the last 30 commits are considered. When false, + * only the most recent commit is considered. Already tested SHAs are skipped + * using commits.json in both cases. * - * @uses passthru() to execute the shell command, which provides direct output - * to the browser. Be aware that using this function with untrusted input can - * lead to security vulnerabilities, such as command injection attacks. + * @param bool $fetch_history Whether to queue the last 30 commits. + * @return string SHA to test, `HEAD` when the SHA could not be resolved, or an empty string when there is nothing new to test. + */ +function select_commit_to_test( $fetch_history ) { + $state = load_commits_state(); + + if ( '' !== $state['testing_commit'] ) { + log_message( 'Resuming tests for wordpress-develop commit ' . $state['testing_commit'] ); + return $state['testing_commit']; + } + + $per_page = $fetch_history ? 30 : 1; + $remote = fetch_wordpress_develop_commit_shas( $per_page ); + + if ( empty( $remote ) ) { + $head = fetch_wordpress_develop_head_sha(); + if ( '' !== $head ) { + $remote = array( $head ); + } + } + + if ( empty( $remote ) ) { + log_message( 'Warning: Could not resolve wordpress-develop commits; falling back to cloned HEAD.' ); + return 'HEAD'; + } + + $candidates = $fetch_history ? array_reverse( $remote ) : array( $remote[0] ); + $known = known_commit_shas( $state ); + + foreach ( $candidates as $candidate ) { + if ( ! in_array( $candidate, $known, true ) ) { + $state['pending_commits'][] = $candidate; + $known[] = $candidate; + } + } + + $state['pending_commits'] = array_values( array_unique( $state['pending_commits'] ) ); + + if ( empty( $state['pending_commits'] ) ) { + save_commits_state( $state ); + return ''; + } + + $sha = array_shift( $state['pending_commits'] ); + $state['testing_commit'] = $sha; + save_commits_state( $state ); + log_message( 'Selected wordpress-develop commit ' . $sha ); + + return $sha; +} + +/** + * Moves the in-progress testing commit to executed_commits after a successful report. * - * @uses error_message() to display an error message if a shell command fails. - * The execution stops at the first failure. + * @return void + */ +function mark_testing_commit_executed() { + $state = load_commits_state(); + if ( '' === $state['testing_commit'] ) { + return; + } + + $state['executed_commits'][] = $state['testing_commit']; + $state['executed_commits'] = array_values( array_unique( $state['executed_commits'] ) ); + $state['testing_commit'] = ''; + save_commits_state( $state ); +} + +/** + * Executes a series of shell commands provided in the operations array. Each operation is logged before execution. + * If any command fails (indicated by a non-zero return code), an error message is displayed. This function is + * useful for automating batch shell tasks within a PHP script, with error handling for each operation. + * + * @param array $operations An array of shell commands (strings) to be executed. Each command should be + * a valid shell command and properly escaped for safety. The commands are executed + * in the order they appear in the array. + * + * @return void This function does not return a value. However, it will output the result of each shell command + * to the standard output and log the execution. It will also halt on the first command that fails, + * displaying an error message. + * + * @uses log_message() to log each operation before execution. This can be used for debugging or auditing purposes. + * @uses passthru() to execute the shell command, which provides direct output to the browser. Be aware that using + * this function with untrusted input can lead to security vulnerabilities, such as command injection attacks. + * @uses error_message() to display an error message if a shell command fails. The execution stops at the first failure. */ function perform_operations( $operations ) { foreach ( $operations as $operation ) { log_message( $operation ); passthru( $operation, $return_code ); - - // Check for command execution failure. if ( 0 !== $return_code ) { error_message( 'Failed to perform operation: ' . $operation . '.' ); - return; } } } /** - * Writes a message to the standard output (STDOUT). - * - * The message is appended with PHP_EOL to ensure proper line breaks on - * different operating systems. + * Writes a message followed by a newline to the standard output (STDOUT). This function is commonly used for logging purposes, + * providing feedback during script execution, or debugging. The message is appended with the PHP end-of-line constant (PHP_EOL) + * to ensure proper line breaks on different operating systems. * - * @param string $message The message to be logged. + * @param string $message The message to be logged. This should be a string, and it will be output exactly as provided, + * followed by a system-specific newline character. * - * @return void This function does not return a value. It directly writes the - * message to STDOUT, which is typically visible in the console or terminal - * where the PHP script is executed. + * @return void This function does not return a value. It directly writes the message to STDOUT, which is typically + * visible in the console or terminal where the PHP script is executed. * - * @uses fwrite() to write the message to STDOUT. This is a low-level file - * operation function that works with various file streams, including standard - * output, standard error, and regular files. + * @uses fwrite() to write the message to STDOUT. This is a low-level file operation function that works with various + * file streams, including standard output, standard error, and regular files. */ function log_message( $message ) { fwrite( STDOUT, $message . PHP_EOL ); } /** - * Displays an error message and terminates the test runner execution. - * - * The error message is prefixed with "Error: " and appended with PHP_EOL - * before being written to the standard output (STDOUT). + * Writes an error message prefixed with "Error: " to the standard error output (STDERR) and terminates the script + * with a status code of 1. This function is typically used to report errors during script execution, where an + * immediate halt is necessary due to unrecoverable conditions. The message is appended with the PHP end-of-line + * constant (PHP_EOL) to ensure it is properly displayed on all operating systems. * - * After outputting the error message, the script will be terminated with a - * status code of 1. + * @param string $message The error message to be logged. This string will be output as provided, but prefixed + * with "Error: " to indicate its nature, followed by a system-specific newline character. * - * @param string $message The error message to be logged. This string will be - * output as provided, but prefixed with "Error: " to indicate its nature, - * followed by a system-specific newline character. + * @return void This function does not return a value. It directly writes the error message to STDERR and then + * terminates the script execution using `exit(1)`, indicating an error condition to the environment. * - * @return void This function does not return a value. It directly writes the - * error message to STDERR and then terminates the script execution using - * `exit(1)`, indicating an error condition to the environment. - * - * @uses fwrite() to write the error message to STDERR. This function is used - * for low-level writing to file streams or output streams, in this case, - * STDERR, which is specifically for error reporting. + * @uses fwrite() to write the error message to STDERR. This function is used for low-level writing to file streams + * or output streams, in this case, STDERR, which is specifically for error reporting. + * @uses exit() to terminate the script execution with a status code of 1, indicating an error has occurred. This is + * a common practice in command-line scripts and applications to signal failure to the calling process or environment. */ function error_message( $message ) { fwrite( STDERR, 'Error: ' . $message . PHP_EOL ); @@ -181,64 +596,47 @@ function error_message( $message ) { } /** - * Ensures a single trailing slash is present at the end of a given string. - * - * File system operations often expect a single trailing slash when referring - * to directories or paths. This ensures that only one trailing slash is - * present at the end of a given string. + * Ensures a single trailing slash is present at the end of a given string. This function first removes any existing + * trailing slashes from the input string to avoid duplication and then appends a single slash. It's commonly used + * to normalize file paths or URLs to ensure consistency in format, especially when concatenating paths or performing + * file system operations that expect a trailing slash. * - * @param string $string The input string to which a trailing slash will be - * added. This could be a file path, URL, or any other string that requires a - * trailing slash for proper formatting or usage. + * @param string $string The input string to which a trailing slash will be added. This could be a file path, URL, + * or any other string that requires a trailing slash for proper formatting or usage. * - * @return string The modified string with a single trailing slash appended at - * the end. If the input string already has one or more trailing slashes, they - * will be trimmed to a single slash. + * @return string The modified string with a single trailing slash appended at the end. If the input string already + * has one or more trailing slashes, they will be trimmed to a single slash. * - * @uses rtrim() to remove any existing trailing slashes from the input string - * before appending a new trailing slash. This ensures that the result - * consistently has exactly one trailing slash, regardless of the input string's - * initial state. + * @uses rtrim() to remove any existing trailing slashes from the input string before appending a new trailing slash. + * This ensures that the result consistently has exactly one trailing slash, regardless of the input string's initial state. */ function trailingslashit( $str ) { return rtrim( $str, '/' ) . '/'; } /** - * Extracts test results from a JUnit XML string. - * - * This extracts the relevant information from the test results into a format - * accepted and understood by the WordPress Test Reporter plugin. - * - * The data specifically extracted is: - * - Total number of tests. - * - Number of failures. - * - Number of errors. - * - Overall execution time. - * - * @param string $xml_string The JUnit XML data as a string. This should be - * well-formed XML representing the results of test executions, typically - * generated by testing frameworks compatible with JUnit reporting. - * - * @return string A JSON encoded string that represents a summary of the test - * results, including overall metrics and detailed information about each failed - * or errored test case. The JSON structure will include keys for 'tests', - * 'failures', 'errors', 'time', and 'testsuites', where 'testsuites' is an - * array of test suites that contains the failures or errors. - * - * @uses simplexml_load_string() to parse the JUnit XML data into an object for - * easy access and manipulation of the XML elements. - * - * @uses xpath() to query specific elements within the XML structure, - * particularly to find test suites with failures or errors. - * - * @uses json_encode() to convert the array structure containing the test - * results into a JSON formatted string. + * Parses JUnit XML formatted string to extract test results, focusing specifically on test failures and errors. + * The function converts the XML data into a structured JSON format that summarizes the overall test outcomes, + * including the total number of tests, failures, errors, and execution time. Only test suites and cases that + * contain failures or errors are included in the final JSON output. This function is useful for automated test + * result analysis, continuous integration reporting, or any scenario where a quick summary of test failures and + * errors is needed. + * + * @param string $xml_string The JUnit XML data as a string. This should be well-formed XML representing the results + * of test executions, typically generated by testing frameworks compatible with JUnit reporting. + * + * @return string A JSON encoded string that represents a summary of the test results, including overall metrics and + * detailed information about each failed or errored test case. The JSON structure will include keys + * for 'tests', 'failures', 'errors', 'time', and 'testsuites', where 'testsuites' is an array of test + * suites that contains the failures or errors. + * + * @uses simplexml_load_string() to parse the JUnit XML data into an object for easy access and manipulation of the XML elements. + * @uses xpath() to query specific elements within the XML structure, particularly to find test suites with failures or errors. + * @uses json_encode() to convert the array structure containing the test results into a JSON formatted string. */ function process_junit_xml( $xml_string ) { if ( empty( $xml_string ) ) { return ''; - return ''; } $xml = simplexml_load_string( $xml_string ); @@ -263,7 +661,6 @@ function process_junit_xml( $xml_string ) { 'failures' => (string) $testsuite['failures'], 'errors' => (string) $testsuite['errors'], ); - if ( empty( $result['failures'] ) && empty( $result['errors'] ) ) { continue; } @@ -289,42 +686,26 @@ function process_junit_xml( $xml_string ) { } /** - * Submits test results to a reporting API endpoint. - * - * This submits test results and related metadata to a site running the - * WordPress Test Reporter plugin using cURL. - * - * Reports are always submitted to WordPress.org Unless the WPT_REPORT_URL - * environment variable is set. - * - * @param string $results The test results in a processed format (e.g., JSON) - * ready for submission to the reporting API. - * - * @param string $rev The SVN revision associated with the test results. This - * often corresponds to a specific code commit or build identifier. - * - * @param string $message The SVN commit message associated with the revision, - * providing context or notes about the changes. - * - * @param string $env The environment data in JSON format, detailing the - * conditions under which the tests were run, such as operating system, PHP - * version, etc. - * - * @param string $api_key The API key for authenticating with the reporting API, - * ensuring secure and authorized access. - * - * @return array An array containing two elements: the HTTP status code of the - * response (int) and the body of the response (string) from the reporting API. - * This can be used to verify successful submission or to handle errors. - * - * @uses curl_init(), curl_setopt(), and curl_exec() to perform the HTTP POST - * request to the reporting API. - * - * @uses json_encode() to encode the data payload as a JSON string for - * submission. - * - * @uses base64_encode() to encode the API key for HTTP Basic Authentication in - * the Authorization header. + * Submits test results along with associated metadata to a specified reporting API. The function constructs + * a POST request containing the test results, SVN revision, SVN message, environment data, and uses an API key + * for authentication. The reporting API's URL is retrieved from an environment variable; if not found, a default + * URL is used. This function is typically used to automate the reporting of test outcomes to a centralized system + * for analysis, tracking, and historical record-keeping. + * + * @param string $results The test results in a processed format (e.g., JSON) ready for submission to the reporting API. + * @param string $rev The SVN revision associated with the test results. This often corresponds to a specific code + * commit or build identifier. + * @param string $message The SVN commit message associated with the revision, providing context or notes about the changes. + * @param string $env The environment data in JSON format, detailing the conditions under which the tests were run, + * such as operating system, PHP version, etc. + * @param string $api_key The API key for authenticating with the reporting API, ensuring secure and authorized access. + * + * @return array An array containing two elements: the HTTP status code of the response (int) and the body of the response + * (string) from the reporting API. This can be used to verify successful submission or to handle errors. + * + * @uses curl_init(), curl_setopt(), and curl_exec() to perform the HTTP POST request to the reporting API. + * @uses json_encode() to encode the data payload as a JSON string for submission. + * @uses base64_encode() to encode the API key for HTTP Basic Authentication in the Authorization header. */ function upload_results( $results, $rev, $message, $env, $api_key ) { $wpt_report_url = getenv( 'WPT_REPORT_URL' ); @@ -341,7 +722,6 @@ function upload_results( $results, $rev, $message, $env, $api_key ) { ); $data_string = json_encode( $data ); - // Set CURL options. curl_setopt( $process, CURLOPT_TIMEOUT, 30 ); curl_setopt( $process, CURLOPT_POST, 1 ); curl_setopt( $process, CURLOPT_CUSTOMREQUEST, 'POST' ); @@ -366,29 +746,26 @@ function upload_results( $results, $rev, $message, $env, $api_key ) { } /** - Collects details about the testing environment. + * Collects and returns an array of key environment details relevant to the application's context. This includes + * the PHP version, installed PHP modules with their versions, system utilities like curl and OpenSSL versions, + * MySQL version, and operating system details. This function is useful for diagnostic purposes, ensuring + * compatibility, or for reporting system configurations in debugging or error logs. * - * The versions of PHP, PHP modules, database software, and system utilities - * can impact the results of the test suite. This gathers the relevant details - * to include in test report submissions. + * The function checks for the availability of specific PHP modules and system utilities and captures their versions. + * It uses shell commands to retrieve system information, which requires the PHP environment to have access to these + * commands and appropriate permissions. * - * @return array An associative array containing detailed environment - * information. The array includes: + * @return array An associative array containing detailed environment information. The array includes: * - 'php_version': The current PHP version. * - 'php_modules': An associative array of selected PHP modules and their versions. - * - 'system_utils': Versions of certain system utilities such as 'curl', 'imagemagick', - * 'graphicsmagick', and 'openssl'. + * - 'system_utils': Versions of certain system utilities such as 'curl', 'imagemagick', 'graphicsmagick', and 'openssl'. * - 'mysql_version': The version of MySQL installed. * - 'os_name': The name of the operating system. * - 'os_version': The version of the operating system. * * @uses phpversion() to get the PHP version and module versions. - * - * @uses shell_exec() to execute system commands for retrieving MySQL version, - * OS details, and versions of utilities like curl and OpenSSL. - * - * @uses class_exists() to check for the availability of the Imagick and Gmagick - * classes for version detection. + * @uses shell_exec() to execute system commands for retrieving MySQL version, OS details, and versions of utilities like curl and OpenSSL. + * @uses class_exists() to check for the availability of the Imagick and Gmagick classes for version detection. */ function get_env_details() { @@ -402,6 +779,7 @@ function get_env_details() { } $env = array( + 'label' => trim( getenv( 'WPT_LABEL' ) ), 'php_version' => phpversion(), 'php_modules' => array(), 'gd_info' => $gd_info, diff --git a/prepare.php b/prepare.php index 9473ca2..9717ccb 100644 --- a/prepare.php +++ b/prepare.php @@ -1,19 +1,17 @@ '$wpt_label', 'php_version' => phpversion(), 'php_modules' => array(), 'gd_info' => \$gd_info, @@ -219,151 +202,134 @@ function curl_selected_bits(\$k) { return in_array(\$k, array('version', 'ssl_ve // Prepend the logger script to the database settings identifier to ensure it gets included in the wp-tests-config.php file. $system_logger = $logger_replace_string . $system_logger; -// Define a string that will set the 'WP_PHP_BINARY' constant to the path of the PHP executable. -$php_binary_string = 'define( \'WP_PHP_BINARY\', \'' . $runner_vars['WPT_PHP_EXECUTABLE'] . '\' );'; +foreach ( $runner_vars['WPT_PHP_EXECUTABLES'] as $php ) { + $paths = get_php_run_paths( $runner_vars, $php ); -/* - * Map configuration file placeholders to environment-specific values. - * - * This is used in the subsequent str_replace operation to replace placeholder - * values in the wp-tests-config-sample.php file with the ones provided. - */ -$wpt_table_prefix = trim( getenv( 'WPT_TABLE_PREFIX' ) ); -$search_replace = array( - 'wptests_' => '' !== $wpt_table_prefix ? $wpt_table_prefix : 'wptests_', - 'youremptytestdbnamehere' => trim( getenv( 'WPT_DB_NAME' ) ), - 'yourusernamehere' => trim( getenv( 'WPT_DB_USER' ) ), - 'yourpasswordhere' => trim( getenv( 'WPT_DB_PASSWORD' ) ), - 'localhost' => trim( getenv( 'WPT_DB_HOST' ) ), - 'define( \'WP_PHP_BINARY\', \'php\' );' => $php_binary_string, - $logger_replace_string => $system_logger, -); + log_message( 'Preparing environment for PHP ' . $php['version'] . ' (' . $php['bin'] . ')' ); -// Replace the placeholders in the wp-tests-config-sample.php file content with actual values. -$contents = str_replace( array_keys( $search_replace ), array_values( $search_replace ), $contents ); + $clone_operations = array(); + if ( is_dir( $paths['prepare_dir'] ) ) { + $clone_operations[] = 'rm -rf ' . escapeshellarg( $paths['prepare_dir'] ); + } -// Write the modified content to the wp-tests-config.php file, which will be used by the test suite. -file_put_contents( $runner_vars['WPT_PREPARE_DIR'] . '/wp-tests-config.php', $contents ); + $clone_operations[] = 'mkdir -p ' . escapeshellarg( $paths['prepare_dir'] ); + $clone_operations[] = 'git clone --depth=1 https://github.com/WordPress/wordpress-develop.git ' . escapeshellarg( $paths['prepare_dir'] ); + $clone_operations[] = 'git -C ' . escapeshellarg( $paths['prepare_dir'] ) . ' config --add safe.directory ' . escapeshellarg( $paths['prepare_dir'] ); -/* - * Construct a command that generates a PHP version string compatible with - * PHPUnit version requirements. - */ -$php_version_cmd = $runner_vars['WPT_PHP_EXECUTABLE'] . " -r \"print PHP_MAJOR_VERSION . '.' . PHP_MINOR_VERSION . '.' . PHP_RELEASE_VERSION;\""; + if ( 'HEAD' !== $commit_sha ) { + $clone_operations[] = 'cd ' . escapeshellarg( $paths['prepare_dir'] ) . ' && git fetch --depth=1 origin ' . escapeshellarg( $commit_sha ) . ' && git checkout ' . escapeshellarg( $commit_sha ); + } -/** - * If an SSH connection string is provided, the command to determine the PHP version is modified - * to execute remotely over SSH. This is required if the test environment is not the local machine. - */ -if ( ! empty( $runner_vars['WPT_SSH_CONNECT'] ) ) { - // The PHP version check command is prefixed with the SSH command, including SSH options, - // and the connection string, ensuring the command is executed on the remote machine. - $php_version_cmd = 'ssh ' . $runner_vars['WPT_SSH_OPTIONS'] . ' ' . escapeshellarg( $runner_vars['WPT_SSH_CONNECT'] ) . ' ' . escapeshellarg( $php_version_cmd ); -} + $clone_operations[] = 'cd ' . escapeshellarg( $paths['prepare_dir'] ) . '; npm install && npm run build'; -// Initialize return value variable for the exec function call. -$retval = 0; + perform_operations( $clone_operations ); -/* - * Execute the constructed command to obtain the PHP version of the test - * environment. - * - * The output is stored in $env_php_version and the return value of the - * command execution is stored in $retval. - */ -$env_php_version = exec( $php_version_cmd, $output, $retval ); + if ( 'HEAD' === $commit_sha ) { + $resolved_sha = trim( (string) exec( 'git --git-dir=' . escapeshellarg( $paths['prepare_dir'] . '/.git' ) . ' rev-parse HEAD' ) ); + if ( '' !== $resolved_sha ) { + $state = load_commits_state(); + if ( '' === $state['testing_commit'] ) { + $state['testing_commit'] = $resolved_sha; + save_commits_state( $state ); + } + $commit_sha = $resolved_sha; + } + } -// Check if the command execution was successful by inspecting the return value. -if ( 0 !== $retval ) { - error_message( 'Could not retrieve the environment PHP Version.' ); -} + log_message( 'Replacing variables in wp-tests-config.php' ); -// Log the obtained PHP version for confirmation and debugging purposes. -log_message( 'Environment PHP Version: ' . $env_php_version ); + $contents = file_get_contents( $paths['prepare_dir'] . '/wp-tests-config-sample.php' ); -/* - * Confirm that the environment meets the minimum PHP version requirement. - * - * When the requirements are not met, execution will end with an error message. - */ -if ( version_compare( $env_php_version, '7.2', '<' ) ) { - // Logs an error message indicating the test runner's incompatibility with PHP versions below 7.2. - error_message( 'The test runner is not compatible with PHP < 7.2.' ); -} + $php_binary_string = 'define( \'WP_PHP_BINARY\', \'' . $php['bin'] . '\' );'; + $wpt_table_prefix = trim( getenv( 'WPT_TABLE_PREFIX' ) ); + $wpt_table_prefix = '' !== $wpt_table_prefix ? $wpt_table_prefix : 'wptests_'; + if ( 'default' !== $php['version'] ) { + $wpt_table_prefix .= str_replace( '.', '_', $php['version'] ) . '_'; + } -// Check if Composer is installed and available in the PATH. -$composer_cmd = 'cd ' . escapeshellarg( $runner_vars['WPT_PREPARE_DIR'] ) . ' && '; -$retval = 0; -$composer_path = escapeshellarg( system( 'which composer', $retval ) ); + $search_replace = array( + 'wptests_' => $wpt_table_prefix, + 'youremptytestdbnamehere' => trim( getenv( 'WPT_DB_NAME' ) ), + 'yourusernamehere' => trim( getenv( 'WPT_DB_USER' ) ), + 'yourpasswordhere' => trim( getenv( 'WPT_DB_PASSWORD' ) ), + 'localhost' => trim( getenv( 'WPT_DB_HOST' ) ), + 'define( \'WP_PHP_BINARY\', \'php\' );' => $php_binary_string, + $logger_replace_string => $system_logger, + ); -if ( 0 === $retval ) { + $contents = str_replace( array_keys( $search_replace ), array_values( $search_replace ), $contents ); - // If Composer is available, prepare the command to use the Composer binary. - $composer_cmd .= $composer_path . ' '; + file_put_contents( $paths['prepare_dir'] . '/wp-tests-config.php', $contents ); -} else { + $php_version_cmd = $php['bin'] . " -r \"print PHP_MAJOR_VERSION . '.' . PHP_MINOR_VERSION . '.' . PHP_RELEASE_VERSION;\""; - // If Composer is not available, download the Composer phar file. - log_message( 'Local Composer not found. Downloading latest stable ...' ); + if ( ! empty( $runner_vars['WPT_SSH_CONNECT'] ) ) { + $php_version_cmd = 'ssh ' . $runner_vars['WPT_SSH_OPTIONS'] . ' ' . escapeshellarg( $runner_vars['WPT_SSH_CONNECT'] ) . ' ' . escapeshellarg( $php_version_cmd ); + } - perform_operations( - array( - 'wget -O ' . escapeshellarg( $runner_vars['WPT_PREPARE_DIR'] . '/composer.phar' ) . ' https://getcomposer.org/composer-stable.phar', - ) - ); + $retval = 0; + $env_php_version = exec( $php_version_cmd, $output, $retval ); - // Update the command to use the downloaded Composer phar file. - $composer_cmd .= $runner_vars['WPT_PHP_EXECUTABLE'] . ' composer.phar '; -} + if ( 0 !== $retval ) { + error_message( 'Could not retrieve the environment PHP Version for ' . $php['version'] . '.' ); + } -// Set the PHP version for Composer to ensure compatibility and update dependencies. -perform_operations( - array( - $composer_cmd . 'config platform.php ' . escapeshellarg( $env_php_version ), - $composer_cmd . 'update', - ) -); + log_message( 'Environment PHP Version: ' . $env_php_version ); -/* - * Transfer the built WordPress codebase to the remote test environment. - * - * When an SSH connection is configured, rsync is used to copy the files - * required to run the WordPress PHPUnit test suite. - * - * The -r option for rsync enables recursive copying to handle nested directory - * structures. - */ -if ( ! empty( $runner_vars['WPT_SSH_CONNECT'] ) ) { - // Initialize rsync options with recursive copying. - $rsync_options = '-r'; + if ( version_compare( $env_php_version, '7.2', '<' ) ) { + error_message( 'The test runner is not compatible with PHP < 7.2.' ); + } + + $composer_cmd = 'cd ' . escapeshellarg( $paths['prepare_dir'] ) . ' && '; + $retval = 0; + $composer_path = escapeshellarg( system( 'which composer', $retval ) ); + + if ( 0 === $retval ) { + $composer_cmd .= $composer_path . ' '; + } else { + log_message( 'Local Composer not found. Downloading latest stable ...' ); + + perform_operations( + array( + 'wget -O ' . escapeshellarg( $paths['prepare_dir'] . '/composer.phar' ) . ' https://getcomposer.org/composer-stable.phar', + ) + ); - // If debug mode is set to verbose, append 'v' to rsync options for verbose output. - if ( $runner_vars['WPT_DEBUG'] ) { - $rsync_options = $rsync_options . 'v'; + $composer_cmd .= $php['bin'] . ' composer.phar '; } - // Perform the rsync operation with the configured options and exclude patterns. - // This operation synchronizes the test environment with the prepared files, excluding - // version control directories and other non-essential files for test execution. perform_operations( array( - 'rsync ' . $rsync_options - . ' --exclude=".git/"' - . ' --exclude="node_modules/"' - . ' --exclude="composer.phar"' - . ' --exclude=".cache/"' - . ' --exclude=".devcontainer/"' - . ' --exclude=".github/"' - . ' --exclude="tools/"' - // Exclude all subdirectories in tests/ except phpunit/. - . ' --exclude="tests/*" --include="tests/phpunit/**"' - . ' -e "ssh ' . $runner_vars['WPT_SSH_OPTIONS'] . '" ' - . escapeshellarg( trailingslashit( $runner_vars['WPT_PREPARE_DIR'] ) ) - . ' ' . escapeshellarg( $runner_vars['WPT_SSH_CONNECT'] . ':' . $runner_vars['WPT_TEST_DIR'] ), + $composer_cmd . 'config platform.php ' . escapeshellarg( $env_php_version ), + $composer_cmd . 'update', ) ); + + if ( ! empty( $runner_vars['WPT_SSH_CONNECT'] ) ) { + $rsync_options = '-r'; + + if ( $runner_vars['WPT_DEBUG'] ) { + $rsync_options = $rsync_options . 'v'; + } + + perform_operations( + array( + 'rsync ' . $rsync_options + . ' --exclude=".git/"' + . ' --exclude="node_modules/"' + . ' --exclude="composer.phar"' + . ' --exclude=".cache/"' + . ' --exclude=".devcontainer/"' + . ' --exclude=".github/"' + . ' --exclude="tools/"' + . ' --exclude="tests/*" --include="tests/phpunit/**"' + . ' -e "ssh ' . $runner_vars['WPT_SSH_OPTIONS'] . '" ' + . escapeshellarg( trailingslashit( $paths['prepare_dir'] ) ) + . ' ' . escapeshellarg( $runner_vars['WPT_SSH_CONNECT'] . ':' . $paths['test_dir'] ), + ) + ); + } } -// Log a success message indicating that the environment has been prepared. log_message( 'Success: Prepared environment.' ); diff --git a/report.php b/report.php index 764ec54..b5f73fd 100644 --- a/report.php +++ b/report.php @@ -1,25 +1,17 @@