diff --git a/dev.yml b/dev.yml index 08378e016..78c37f7e4 100644 --- a/dev.yml +++ b/dev.yml @@ -39,6 +39,10 @@ up: name: Install the pinned Maestro version met?: ./scripts/install_maestro --check meet: ./scripts/install_maestro + - custom: + name: Install the pinned maestro-runner version + met?: ./scripts/install_maestro_runner --check + meet: ./scripts/install_maestro_runner - custom: name: Run Checkout Kit workspace setup met?: ./scripts/setup_dev_workspace --check --skip-optional-prompts diff --git a/e2e/.maestro-runner-checksums b/e2e/.maestro-runner-checksums new file mode 100644 index 000000000..c11ac9b2f --- /dev/null +++ b/e2e/.maestro-runner-checksums @@ -0,0 +1,4 @@ +58ffe553995b960606bd47853d205efd65fa91ae7444bd1d976453ecdedde997 maestro-runner-1.1.24-darwin-amd64.tar.gz +2771af3a20aa479a4fd948afab87092daba05c7fed454e8306726a804a288e03 maestro-runner-1.1.24-darwin-arm64.tar.gz +9057c9f1beb87d7b1e92b4d6edadab70e85f39603cd3ed4e690a4ce00ed44563 maestro-runner-1.1.24-linux-amd64.tar.gz +ab357a5d0ac40f79b53f60d862d88fed96e134d314354fb6a898210aa8496ee5 maestro-runner-1.1.24-linux-arm64.tar.gz diff --git a/e2e/.maestro-runner-version b/e2e/.maestro-runner-version new file mode 100644 index 000000000..3fe3e58a8 --- /dev/null +++ b/e2e/.maestro-runner-version @@ -0,0 +1 @@ +1.1.24 diff --git a/e2e/README.md b/e2e/README.md index 160c2b568..20aa612b2 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -138,6 +138,21 @@ Count BrowserStack run plan rows: ruby e2e/scripts/e2e_matrix_to_browserstack_run_plan count ``` +## BrowserStack executors + +The Bitrise run stage defaults to BrowserStack-hosted Maestro. Set +`E2E_BROWSERSTACK_EXECUTOR=maestro-runner` to use the checksum-verified pin in +`.maestro-runner-version`; it runs on the Bitrise worker and connects to BrowserStack +real devices through the Appium hub. + +The Appium executor uploads only the app artifact, writes the runner's JSON, JUnit, +HTML, screenshots, hierarchy, logs, and session IDs into the run result artifact, +and preserves the normalized `result.json` contract used by GitHub reporting. It +uses a temporary copy of the E2E workspace to enable `launchApp.newSession` on +Android. On iOS it relies on BrowserStack's clean physical-device session and skips +the simulator-only `clearState` command that BrowserStack cannot execute. The +shared flows consumed by hosted Maestro remain unchanged. + ## Files - `config.yaml` configures Maestro for shared platform behavior and quarantines diff --git a/e2e/bitrise.yml b/e2e/bitrise.yml index 93c45b974..83b248d5e 100644 --- a/e2e/bitrise.yml +++ b/e2e/bitrise.yml @@ -404,16 +404,33 @@ workflows: envman add --key E2E_BROWSERSTACK_RESULTS_DIR --value "$results_dir" : "${BROWSERSTACK_USERNAME:?BROWSERSTACK_USERNAME is required}" : "${BROWSERSTACK_ACCESS_KEY:?BROWSERSTACK_ACCESS_KEY is required}" - : "${E2E_TESTS_ZIP:?E2E_TESTS_ZIP is required. Check e2e-execute-browserstack-run pull-intermediate-files artifact_sources.}" : "${E2E_BROWSERSTACK_RUN_PLAN_JSON:?E2E_BROWSERSTACK_RUN_PLAN_JSON is required. Check e2e-execute-browserstack-run pull-intermediate-files artifact_sources.}" run_artifact_env="$(ruby -rjson -e 'run_plan = JSON.parse(File.read(ARGV.fetch(0))); puts run_plan.fetch(Integer(ARGV.fetch(1))).fetch("artifact_env")' "$E2E_BROWSERSTACK_RUN_PLAN_JSON" "$run_index")" : "${!run_artifact_env:?${run_artifact_env} is required. Check e2e-execute-browserstack-run pull-intermediate-files artifact_sources.}" - e2e_log "Executing BrowserStack run plan row ${run_index}" - e2e/scripts/execute_browserstack_run \ - --index "$run_index" \ - --run-plan "$E2E_BROWSERSTACK_RUN_PLAN_JSON" \ - --tests-zip "$E2E_TESTS_ZIP" \ - --output-dir "$results_dir" + executor="${E2E_BROWSERSTACK_EXECUTOR:-hosted-maestro}" + case "$executor" in + maestro-runner) + scripts/install_maestro_runner + e2e_log "Executing BrowserStack run plan row ${run_index} with maestro-runner" + e2e/scripts/execute_browserstack_runner_run \ + --index "$run_index" \ + --run-plan "$E2E_BROWSERSTACK_RUN_PLAN_JSON" \ + --output-dir "$results_dir" + ;; + hosted-maestro) + : "${E2E_TESTS_ZIP:?E2E_TESTS_ZIP is required for hosted-maestro}" + e2e_log "Executing BrowserStack run plan row ${run_index} with hosted Maestro" + e2e/scripts/execute_browserstack_run \ + --index "$run_index" \ + --run-plan "$E2E_BROWSERSTACK_RUN_PLAN_JSON" \ + --tests-zip "$E2E_TESTS_ZIP" \ + --output-dir "$results_dir" + ;; + *) + e2e_log "Unsupported E2E_BROWSERSTACK_EXECUTOR: $executor" + exit 1 + ;; + esac - deploy-to-bitrise-io@2: is_always_run: true inputs: diff --git a/e2e/lib/browserstack_client.rb b/e2e/lib/browserstack_client.rb index 251a05143..e31ccd958 100644 --- a/e2e/lib/browserstack_client.rb +++ b/e2e/lib/browserstack_client.rb @@ -50,6 +50,16 @@ def upload(path, file_path, custom_id) @client.execute(request) end + def upload_app_automate_app(file_path, custom_id) + upload("/app-automate/upload", file_path, custom_id) + end + + def find_build_by_name(name) + builds = @client.get("/automate/builds.json?limit=100") + match = builds.find { |entry| entry.dig("automation_build", "name") == name } + match&.fetch("automation_build") + end + def start_build(platform, body) @client.post_json("/app-automate/maestro/v2/#{platform}/build", body) end diff --git a/e2e/lib/browserstack_runner_executor.rb b/e2e/lib/browserstack_runner_executor.rb new file mode 100644 index 000000000..000571822 --- /dev/null +++ b/e2e/lib/browserstack_runner_executor.rb @@ -0,0 +1,275 @@ +# frozen_string_literal: true + +require "fileutils" +require "json" +require "open3" +require "tempfile" +require "time" +require "tmpdir" +require_relative "browserstack_client" +require_relative "browserstack_device_resolver" + +class BrowserStackRunnerExecutor + APPIUM_URL = "https://hub-cloud.browserstack.com/wd/hub" + + def initialize(client:, run:, app_path:, output_dir:, runner_binary:, env: ENV, workspace: File.expand_path("..", __dir__)) + @client = client + @run = run + @app_path = app_path + @output_dir = output_dir + @runner_binary = runner_binary + @env = env + @workspace = workspace + end + + def execute! + validate! + FileUtils.mkdir_p(@output_dir) + + resolved_device = resolve_device + app_url = @client.upload_app_automate_app(@app_path, app_custom_id).fetch("app_url") + build_name = browserstack_build_name + runner_output = File.join(@output_dir, "maestro-runner") + sessions_path = File.join(@output_dir, "appium-sessions.jsonl") + report = nil + runner_passed = false + + with_runner_workspace do |workspace| + Tempfile.create(["checkout-kit-appium-capabilities", ".json"]) do |file| + file.write(JSON.generate(capabilities(@run, resolved_device, app_url, build_name, username, access_key))) + file.flush + command = runner_command(@runner_binary, file.path, runner_output, sessions_path, @run) + runner_passed = run_process(command, File.join(@output_dir, "maestro-runner.log"), workspace) + report = read_report(runner_output) + end + end + + build_id = find_build_id(build_name) + result = normalized_result(@run, resolved_device, report, build_id) + result["passed"] &&= runner_passed + write_result(result) + result.fetch("passed") + rescue StandardError => error + write_result(failure_result(error)) + warn "BrowserStack maestro-runner execution failed: #{error.message}" + false + ensure + redact_artifacts + end + + private + + def validate! + raise "BrowserStack username is required" if username.empty? + raise "BrowserStack access key is required" if access_key.empty? + raise "application artifact does not exist: #{@app_path}" unless File.file?(@app_path) + raise "maestro-runner is not executable: #{@runner_binary}" unless File.executable?(@runner_binary) + end + + def username + @env.fetch("BROWSERSTACK_USERNAME", "") + end + + def access_key + @env.fetch("BROWSERSTACK_ACCESS_KEY", "") + end + + def resolve_device + devices = @client.list_devices + limits = @client.list_device_tier_limits + BrowserStackDeviceResolver.new(devices, limits).resolve(@run.fetch("device_selector")) + end + + def app_custom_id + commit = @env["GIT_COMMIT"] || @env["BITRISE_GIT_COMMIT"] || "local" + "checkout-kit-#{@run.fetch("target")}-#{@run.fetch("platform")}-#{commit}".gsub(/[^a-zA-Z0-9_.-]/, "-") + end + + def browserstack_build_name + identifier = @env["BITRISE_BUILD_NUMBER"] || @env["GITHUB_RUN_ID"] || @env["GIT_COMMIT"] || Time.now.utc.strftime("%Y%m%d%H%M%S") + "checkout-kit maestro-runner #{identifier} #{@run.fetch("id")}" + end + + def find_build_id(build_name) + @client.find_build_by_name(build_name)&.fetch("hashed_id", nil) + rescue StandardError => error + warn "Unable to resolve BrowserStack build #{build_name}: #{error.message}" + nil + end + + def capabilities(run, resolved_device, app_url, build_name, browserstack_username, browserstack_access_key) + platform = run.fetch("platform") + result = { + "platformName" => platform == "ios" ? "iOS" : "Android", + "appium:automationName" => platform == "ios" ? "XCUITest" : "UiAutomator2", + "appium:app" => app_url, + "appium:deviceName" => resolved_device.fetch("resolved_device"), + "appium:platformVersion" => resolved_device.fetch("resolved_os_version"), + "appium:noReset" => false, + "appium:fullReset" => false, + "appium:newCommandTimeout" => 300, + "bstack:options" => { + "userName" => browserstack_username, + "accessKey" => browserstack_access_key, + "projectName" => "Checkout Kit E2E", + "buildName" => build_name, + "sessionName" => run.fetch("id"), + "debug" => true, + "video" => true, + "deviceLogs" => true, + "networkLogs" => true, + "idleTimeout" => 300 + } + } + + if platform == "ios" + result["appium:bundleId"] = run.fetch("app_id") + else + result["appium:appPackage"] = run.fetch("app_id") + result["appium:autoGrantPermissions"] = true + end + + result + end + + def runner_command(binary, caps_path, runner_output, sessions_path, run) + command = [ + binary, + "--driver", "appium", + "--platform", run.fetch("platform"), + "--appium-url", APPIUM_URL, + "--caps", caps_path, + "--appium-session-file", sessions_path, + "test", + "--output", runner_output, + "--flatten", + "--artifacts", "always" + ] + + comma_separated(run["include_tags"]).each { |tag| command.concat(["--include-tags", tag]) } + comma_separated(run["exclude_tags"]).each { |tag| command.concat(["--exclude-tags", tag]) } + flow_environment(run).each { |key, value| command.concat(["--env", "#{key}=#{value}"]) } + command.concat(test_paths(run)) + end + + def test_paths(run) + paths = ["tests/shared"] + target_path = "tests/#{run.fetch("target")}" + paths << target_path if @workspace && File.directory?(File.join(@workspace, target_path)) + paths + end + + def comma_separated(value) + values = value.is_a?(Array) ? value : value.to_s.split(",") + values.map(&:to_s).map(&:strip).reject(&:empty?) + end + + def flow_environment(run) + { + "E2E_APP_ID" => run.fetch("app_id"), + "E2E_READY_MARKER" => run.fetch("ready_marker"), + "E2E_CONTROL_LINK" => run.fetch("control_link") + }.merge(run.fetch("env", {})) + end + + def with_runner_workspace + Dir.mktmpdir("checkout-kit-maestro-runner") do |workspace| + FileUtils.cp_r(File.join(@workspace, "."), workspace) + launch_path = File.join(workspace, "flows/app/launch.yaml") + launch_flow = File.read(launch_path) + replacement = if @run.fetch("platform") == "ios" + "\\1clearState: false" + else + "\\1clearState: true\n\\1newSession: true" + end + updated_flow = launch_flow.sub(/^(\s*)clearState: true$/, replacement) + raise "could not adapt Appium launch in #{launch_path}" if updated_flow == launch_flow + + File.write(launch_path, updated_flow) + yield workspace + end + end + + def run_process(command, log_path, workspace) + success = false + File.open(log_path, "w") do |log| + Open3.popen2e(*command, chdir: workspace) do |_stdin, output, wait_thread| + output.each do |line| + safe_line = redact(line) + $stdout.write(safe_line) + log.write(safe_line) + end + success = wait_thread.value.success? + end + end + success + end + + def redact(value) + [username, access_key].reject(&:empty?).reduce(value) { |text, secret| text.gsub(secret, "[REDACTED]") } + end + + def redact_artifacts + return unless @output_dir && File.directory?(@output_dir) + + Dir.glob(File.join(@output_dir, "**", "*.{html,json,jsonl,log,txt,xml}")).each do |path| + contents = File.binread(path) + redacted = redact(contents) + File.binwrite(path, redacted) if redacted != contents + end + end + + def read_report(runner_output) + path = File.join(runner_output, "report.json") + raise "maestro-runner did not produce #{path}" unless File.file?(path) + + JSON.parse(File.read(path)) + end + + def normalized_result(run, resolved_device, report, build_id) + failed_tests = report.fetch("flows", []).filter_map do |flow| + next unless flow["status"] == "failed" + + { + "name" => flow.fetch("name", "unknown"), + "source_file" => flow["sourceFile"], + "duration_ms" => flow["duration"], + "error" => flow["error"] + }.compact + end + + { + "id" => run.fetch("id"), + "target" => run.fetch("target"), + "application_id" => run.fetch("application_id"), + "platform" => run.fetch("platform"), + "os_version_tag" => run.fetch("os_version_tag"), + "resolved_device" => resolved_device.fetch("resolved_device"), + "resolved_os_version" => resolved_device.fetch("resolved_os_version"), + "passed" => report["status"] == "passed", + "duration_ms" => report.fetch("flows", []).sum { |flow| flow.fetch("duration", 0).to_i }, + "summary" => report.fetch("summary", {}), + "failed_tests" => failed_tests, + "build_id" => build_id, + "executor" => "maestro-runner" + } + end + + def failure_result(error) + { + "id" => @run["id"], + "target" => @run["target"], + "application_id" => @run["application_id"], + "platform" => @run["platform"], + "os_version_tag" => @run["os_version_tag"], + "passed" => false, + "failed_tests" => [{"name" => "maestro-runner setup", "error" => error.message}], + "executor" => "maestro-runner" + }.compact + end + + def write_result(result) + FileUtils.mkdir_p(@output_dir) + File.write(File.join(@output_dir, "result.json"), JSON.pretty_generate(result)) + end +end diff --git a/e2e/scripts/execute_browserstack_runner_run b/e2e/scripts/execute_browserstack_runner_run new file mode 100755 index 000000000..fe2f06930 --- /dev/null +++ b/e2e/scripts/execute_browserstack_runner_run @@ -0,0 +1,41 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "json" +require "optparse" +require_relative "../lib/browserstack_client" +require_relative "../lib/browserstack_runner_executor" + +options = { + run_plan_path: ENV["E2E_BROWSERSTACK_RUN_PLAN_JSON"] || "browserstack-run-plan.json", + run_index: ENV["BITRISE_IO_PARALLEL_INDEX"]&.to_i, + output_dir: ENV["E2E_BROWSERSTACK_RESULTS_DIR"] || File.join(Dir.pwd, "e2e-results") +} + +OptionParser.new do |parser| + parser.on("--run-plan PATH") { |path| options[:run_plan_path] = path } + parser.on("--index INDEX", Integer) { |index| options[:run_index] = index } + parser.on("--output-dir PATH") { |path| options[:output_dir] = path } +end.parse! + +raise ArgumentError, "run index is required" if options[:run_index].nil? + +run_plan = JSON.parse(File.read(options.fetch(:run_plan_path))) +run = run_plan.fetch(options.fetch(:run_index)) +app_path = ENV.fetch(run.fetch("artifact_env")) +runner_binary = `#{File.expand_path("maestro_runner_bin", __dir__)}`.strip +raise "could not resolve maestro-runner" unless $?.success? + +client = BrowserStackClient.new( + username: ENV.fetch("BROWSERSTACK_USERNAME"), + access_key: ENV.fetch("BROWSERSTACK_ACCESS_KEY"), + retries: ENV.fetch("E2E_BROWSERSTACK_API_RETRIES", "1").to_i +) + +BrowserStackRunnerExecutor.new( + client: client, + run: run, + app_path: app_path, + output_dir: options.fetch(:output_dir), + runner_binary: runner_binary +).execute! diff --git a/e2e/scripts/maestro_runner_bin b/e2e/scripts/maestro_runner_bin new file mode 100755 index 000000000..631d1e7de --- /dev/null +++ b/e2e/scripts/maestro_runner_bin @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail + +E2E_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +VERSIONS_ROOT="${MAESTRO_RUNNER_VERSIONS_ROOT:-$HOME/.maestro-runner-versions}" +version="$(tr -d '[:space:]' <"$E2E_ROOT/.maestro-runner-version")" +binary="$VERSIONS_ROOT/$version/maestro-runner" + +if [ ! -x "$binary" ]; then + echo "maestro_runner_bin: maestro-runner $version is not installed at $binary." >&2 + echo " run: scripts/install_maestro_runner" >&2 + exit 1 +fi + +printf '%s\n' "$binary" diff --git a/e2e/test/browserstack_client_test.rb b/e2e/test/browserstack_client_test.rb index ff14f0362..c3c92f0d7 100644 --- a/e2e/test/browserstack_client_test.rb +++ b/e2e/test/browserstack_client_test.rb @@ -4,6 +4,13 @@ require_relative "../lib/browserstack_client" class BrowserStackClientTest < Minitest::Test + FakeHttpClient = Struct.new(:response, :path) do + def get(request_path) + self.path = request_path + response + end + end + def test_build_url_appends_build_id assert_equal "https://app-automate.browserstack.com/dashboard/v2/builds/abc123", BrowserStackClient.build_url("abc123") end @@ -12,4 +19,31 @@ def test_build_url_without_build_id_returns_dashboard_base assert_equal "https://app-automate.browserstack.com/dashboard/v2/builds", BrowserStackClient.build_url(nil) assert_equal "https://app-automate.browserstack.com/dashboard/v2/builds", BrowserStackClient.build_url("") end + + def test_standard_app_upload_uses_the_app_automate_endpoint + client = BrowserStackClient.allocate + client.define_singleton_method(:upload) do |path, file_path, custom_id| + {"path" => path, "file_path" => file_path, "custom_id" => custom_id} + end + + response = client.upload_app_automate_app("sample.ipa", "checkout-kit-ios-abc") + + assert_equal "/app-automate/upload", response.fetch("path") + assert_equal "sample.ipa", response.fetch("file_path") + assert_equal "checkout-kit-ios-abc", response.fetch("custom_id") + end + + def test_find_build_by_name_returns_the_standard_app_automate_build + http = FakeHttpClient.new([ + {"automation_build" => {"name" => "other", "hashed_id" => "first"}}, + {"automation_build" => {"name" => "checkout-kit maestro-runner 42", "hashed_id" => "wanted"}} + ]) + client = BrowserStackClient.allocate + client.instance_variable_set(:@client, http) + + build = client.find_build_by_name("checkout-kit maestro-runner 42") + + assert_equal "wanted", build.fetch("hashed_id") + assert_equal "/automate/builds.json?limit=100", http.path + end end diff --git a/e2e/test/browserstack_runner_executor_test.rb b/e2e/test/browserstack_runner_executor_test.rb new file mode 100644 index 000000000..3f345e034 --- /dev/null +++ b/e2e/test/browserstack_runner_executor_test.rb @@ -0,0 +1,205 @@ +# frozen_string_literal: true + +require "minitest/autorun" +require "tmpdir" +require_relative "../lib/browserstack_runner_executor" + +class BrowserStackRunnerExecutorTest < Minitest::Test + def setup + @executor = BrowserStackRunnerExecutor.allocate + end + + def test_android_capabilities_install_the_uploaded_app_on_the_resolved_device + capabilities = @executor.send( + :capabilities, + { + "id" => "kotlin-android-latest", + "application_id" => "kotlin-android", + "app_id" => "com.shopify.checkoutkit", + "platform" => "android" + }, + { + "resolved_device" => "Google Pixel 9", + "resolved_os_version" => "15.0" + }, + "bs://uploaded-app", + "checkout-kit maestro-runner 123", + "user", + "key" + ) + + assert_equal "Android", capabilities.fetch("platformName") + assert_equal "UiAutomator2", capabilities.fetch("appium:automationName") + assert_equal "bs://uploaded-app", capabilities.fetch("appium:app") + assert_equal "Google Pixel 9", capabilities.fetch("appium:deviceName") + assert_equal "15.0", capabilities.fetch("appium:platformVersion") + assert_equal "com.shopify.checkoutkit", capabilities.fetch("appium:appPackage") + assert_equal false, capabilities.fetch("appium:noReset") + assert_equal true, capabilities.fetch("appium:autoGrantPermissions") + + browserstack = capabilities.fetch("bstack:options") + assert_equal "user", browserstack.fetch("userName") + assert_equal "key", browserstack.fetch("accessKey") + assert_equal "checkout-kit maestro-runner 123", browserstack.fetch("buildName") + assert_equal "kotlin-android-latest", browserstack.fetch("sessionName") + refute browserstack.key?("appiumVersion") + refute browserstack.key?("language") + refute browserstack.key?("locale") + end + + def test_ios_capabilities_use_xcuitest_and_the_bundle_identifier + capabilities = @executor.send( + :capabilities, + { + "id" => "swift-ios-latest", + "application_id" => "swift-ios", + "app_id" => "com.shopify.checkoutkit", + "platform" => "ios" + }, + { + "resolved_device" => "iPhone 16", + "resolved_os_version" => "18" + }, + "bs://uploaded-app", + "checkout-kit maestro-runner abc", + "user", + "key" + ) + + assert_equal "iOS", capabilities.fetch("platformName") + assert_equal "XCUITest", capabilities.fetch("appium:automationName") + assert_equal "com.shopify.checkoutkit", capabilities.fetch("appium:bundleId") + refute capabilities.key?("appium:autoGrantPermissions") + end + + def test_command_translates_tags_and_environment_to_repeated_runner_flags + command = @executor.send( + :runner_command, + "/tmp/maestro-runner", + "/tmp/caps.json", + "/tmp/results", + "/tmp/sessions.jsonl", + { + "platform" => "ios", + "target" => "swift", + "app_id" => "com.shopify.checkoutkit", + "ready_marker" => "sample-ready", + "control_link" => "com.shopify.checkoutkit://e2e", + "include_tags" => ["launch", "checkout"], + "exclude_tags" => ["android", "local"], + "env" => {"CART" => "synthetic-cart", "EMPTY" => ""} + } + ) + + assert_equal [ + "/tmp/maestro-runner", + "--driver", "appium", + "--platform", "ios", + "--appium-url", BrowserStackRunnerExecutor::APPIUM_URL, + "--caps", "/tmp/caps.json", + "--appium-session-file", "/tmp/sessions.jsonl", + "test", + "--output", "/tmp/results", + "--flatten", + "--artifacts", "always", + "--include-tags", "launch", + "--include-tags", "checkout", + "--exclude-tags", "android", + "--exclude-tags", "local", + "--env", "E2E_APP_ID=com.shopify.checkoutkit", + "--env", "E2E_READY_MARKER=sample-ready", + "--env", "E2E_CONTROL_LINK=com.shopify.checkoutkit://e2e", + "--env", "CART=synthetic-cart", + "--env", "EMPTY=", + "tests/shared" + ], command + end + + def test_redaction_removes_browserstack_credentials_from_runner_output + @executor.instance_variable_set(:@env, { + "BROWSERSTACK_USERNAME" => "browserstack-user", + "BROWSERSTACK_ACCESS_KEY" => "browserstack-key" + }) + + redacted = @executor.send(:redact, "user=browserstack-user key=browserstack-key") + + assert_equal "user=[REDACTED] key=[REDACTED]", redacted + end + + def test_runner_workspace_enables_a_fresh_appium_session_without_changing_the_shared_flow + Dir.mktmpdir do |source| + launch_path = File.join(source, "flows/app/launch.yaml") + FileUtils.mkdir_p(File.dirname(launch_path)) + File.write(launch_path, "- launchApp:\n clearState: true\n arguments: {}\n") + @executor.instance_variable_set(:@workspace, source) + @executor.instance_variable_set(:@run, {"platform" => "android"}) + + @executor.send(:with_runner_workspace) do |workspace| + copied_launch = File.read(File.join(workspace, "flows/app/launch.yaml")) + assert_includes copied_launch, "clearState: true\n newSession: true" + end + + refute_includes File.read(launch_path), "newSession" + end + end + + def test_ios_runner_workspace_skips_the_simulator_only_clear_state_command + Dir.mktmpdir do |source| + launch_path = File.join(source, "flows/app/launch.yaml") + FileUtils.mkdir_p(File.dirname(launch_path)) + File.write(launch_path, "- launchApp:\n clearState: true\n arguments: {}\n") + @executor.instance_variable_set(:@workspace, source) + @executor.instance_variable_set(:@run, {"platform" => "ios"}) + + @executor.send(:with_runner_workspace) do |workspace| + copied_launch = File.read(File.join(workspace, "flows/app/launch.yaml")) + assert_includes copied_launch, "clearState: false" + refute_includes copied_launch, "newSession" + end + + assert_includes File.read(launch_path), "clearState: true" + end + end + + def test_normalized_result_uses_runner_report_for_failures_and_timing + result = @executor.send( + :normalized_result, + { + "id" => "react-native-ios-latest", + "target" => "react-native-ios", + "application_id" => "com.shopify.checkoutkit", + "platform" => "ios", + "os_version_tag" => "latest" + }, + { + "resolved_device" => "iPhone 16", + "resolved_os_version" => "18" + }, + { + "status" => "failed", + "startTime" => "2026-03-10T10:00:00Z", + "endTime" => "2026-03-10T10:01:02.500Z", + "summary" => {"total" => 2, "passed" => 1, "failed" => 1}, + "flows" => [ + {"name" => "Launch smoke", "status" => "passed", "duration" => 12_000}, + {"name" => "Guest checkout", "sourceFile" => "tests/shared/checkout-guest.yaml", "status" => "failed", "duration" => 50_500, "error" => "Card field timed out"} + ] + }, + "build-hash" + ) + + assert_equal false, result.fetch("passed") + assert_equal 62_500, result.fetch("duration_ms") + assert_equal({"total" => 2, "passed" => 1, "failed" => 1}, result.fetch("summary")) + assert_equal [ + { + "name" => "Guest checkout", + "source_file" => "tests/shared/checkout-guest.yaml", + "duration_ms" => 50_500, + "error" => "Card field timed out" + } + ], result.fetch("failed_tests") + assert_equal "build-hash", result.fetch("build_id") + assert_equal "maestro-runner", result.fetch("executor") + end +end diff --git a/e2e/test/maestro_runner_installation_test.rb b/e2e/test/maestro_runner_installation_test.rb new file mode 100644 index 000000000..2e8429429 --- /dev/null +++ b/e2e/test/maestro_runner_installation_test.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +require "minitest/autorun" +require "open3" +require "tmpdir" + +class MaestroRunnerInstallationTest < Minitest::Test + REPO_ROOT = File.expand_path("../..", __dir__) + VERSION_FILE = File.join(REPO_ROOT, "e2e/.maestro-runner-version") + CHECKSUMS_FILE = File.join(REPO_ROOT, "e2e/.maestro-runner-checksums") + INSTALLER = File.join(REPO_ROOT, "scripts/install_maestro_runner") + RESOLVER = File.join(REPO_ROOT, "e2e/scripts/maestro_runner_bin") + + def test_release_is_pinned_with_checksums_for_every_supported_worker + version = File.read(VERSION_FILE).strip + checksums = File.readlines(CHECKSUMS_FILE, chomp: true).to_h { |line| line.split(" ", 2).reverse } + + assert_equal "1.1.24", version + %w[darwin-amd64 darwin-arm64 linux-amd64 linux-arm64].each do |platform| + asset = "maestro-runner-#{version}-#{platform}.tar.gz" + assert_match(/\A[0-9a-f]{64}\z/, checksums.fetch(asset)) + end + end + + def test_resolver_returns_the_pinned_executable + Dir.mktmpdir do |versions_root| + version = File.read(VERSION_FILE).strip + binary = File.join(versions_root, version, "maestro-runner") + FileUtils.mkdir_p(File.dirname(binary)) + File.write(binary, "#!/bin/sh\nexit 0\n") + File.chmod(0o755, binary) + + stdout, stderr, status = Open3.capture3({"MAESTRO_RUNNER_VERSIONS_ROOT" => versions_root}, RESOLVER) + + assert status.success?, stderr + assert_equal "#{binary}\n", stdout + end + end +end diff --git a/scripts/install_maestro_runner b/scripts/install_maestro_runner new file mode 100755 index 000000000..92f264e21 --- /dev/null +++ b/scripts/install_maestro_runner @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +VERSIONS_ROOT="${MAESTRO_RUNNER_VERSIONS_ROOT:-$HOME/.maestro-runner-versions}" +version="$(tr -d '[:space:]' <"$REPO_ROOT/e2e/.maestro-runner-version")" +destination="$VERSIONS_ROOT/$version" +binary="$destination/maestro-runner" +mode="install" + +if [ "${1:-}" = "--check" ]; then + mode="check" +elif [ "$#" -ne 0 ]; then + echo "Usage: scripts/install_maestro_runner [--check]" >&2 + exit 1 +fi + +installed_version() { + if [ ! -x "$binary" ]; then + return 1 + fi + "$binary" --version 2>/dev/null | head -1 | awk '{print $2}' +} + +if [ "$mode" = "check" ]; then + found="$(installed_version || true)" + if [ "$found" != "$version" ]; then + echo "install_maestro_runner: expected $version at $binary, found ${found:-nothing}." >&2 + exit 1 + fi + exit 0 +fi + +if [ "$(installed_version || true)" = "$version" ]; then + echo "maestro-runner $version is installed at $binary" + exit 0 +fi + +case "$(uname -s)" in + Darwin) os="darwin" ;; + Linux) os="linux" ;; + *) echo "install_maestro_runner: unsupported OS $(uname -s)" >&2; exit 1 ;; +esac + +case "$(uname -m)" in + x86_64|amd64) arch="amd64" ;; + arm64|aarch64) arch="arm64" ;; + *) echo "install_maestro_runner: unsupported architecture $(uname -m)" >&2; exit 1 ;; +esac + +asset="maestro-runner-$version-$os-$arch.tar.gz" +expected_checksum="$(awk -v asset="$asset" '$2 == asset { print $1 }' "$REPO_ROOT/e2e/.maestro-runner-checksums")" +if [ -z "$expected_checksum" ]; then + echo "install_maestro_runner: no checksum pinned for $asset" >&2 + exit 1 +fi + +scratch="$(mktemp -d "${TMPDIR:-/tmp}/checkout-kit-maestro-runner.XXXXXX")" +trap 'rm -rf "$scratch"' EXIT +archive="$scratch/$asset" +curl -fsSL "https://github.com/devicelab-dev/maestro-runner/releases/download/v$version/$asset" -o "$archive" +if command -v shasum >/dev/null 2>&1; then + actual_checksum="$(shasum -a 256 "$archive" | awk '{print $1}')" +else + actual_checksum="$(sha256sum "$archive" | awk '{print $1}')" +fi +if [ "$actual_checksum" != "$expected_checksum" ]; then + echo "install_maestro_runner: checksum mismatch for $asset" >&2 + exit 1 +fi + +mkdir -p "$destination" +tar -xzf "$archive" -C "$scratch" +extracted="$(find "$scratch" -type f -name maestro-runner -perm -111 | head -1)" +if [ -z "$extracted" ]; then + echo "install_maestro_runner: $asset did not contain an executable" >&2 + exit 1 +fi +install -m 755 "$extracted" "$binary" + +if [ "$(installed_version || true)" != "$version" ]; then + echo "install_maestro_runner: $binary does not report $version" >&2 + exit 1 +fi + +echo "maestro-runner $version is installed at $binary"