From d24d1204ba8d9f9b7b6ce1aba50d62a92c05f5f8 Mon Sep 17 00:00:00 2001 From: Kieran Osgood Date: Thu, 20 Aug 2026 19:20:49 +0100 Subject: [PATCH 1/5] Prototype maestro-runner on BrowserStack Appium --- dev.yml | 4 + e2e/.maestro-runner-checksums | 4 + e2e/.maestro-runner-version | 1 + e2e/README.md | 14 + e2e/bitrise.yml | 31 ++- e2e/lib/browserstack_client.rb | 10 + e2e/lib/browserstack_runner_executor.rb | 256 ++++++++++++++++++ e2e/scripts/execute_browserstack_runner_run | 41 +++ e2e/scripts/maestro_runner_bin | 15 + e2e/test/browserstack_client_test.rb | 34 +++ e2e/test/browserstack_runner_executor_test.rb | 174 ++++++++++++ e2e/test/maestro_runner_installation_test.rb | 39 +++ scripts/install_maestro_runner | 86 ++++++ 13 files changed, 702 insertions(+), 7 deletions(-) create mode 100644 e2e/.maestro-runner-checksums create mode 100644 e2e/.maestro-runner-version create mode 100644 e2e/lib/browserstack_runner_executor.rb create mode 100755 e2e/scripts/execute_browserstack_runner_run create mode 100755 e2e/scripts/maestro_runner_bin create mode 100644 e2e/test/browserstack_runner_executor_test.rb create mode 100644 e2e/test/maestro_runner_installation_test.rb create mode 100755 scripts/install_maestro_runner diff --git a/dev.yml b/dev.yml index 08378e016..78c37f7e4 100644 --- a/dev.yml +++ b/dev.yml @@ -39,6 +39,10 @@ up: name: Install the pinned Maestro version met?: ./scripts/install_maestro --check meet: ./scripts/install_maestro + - custom: + name: Install the pinned maestro-runner version + met?: ./scripts/install_maestro_runner --check + meet: ./scripts/install_maestro_runner - custom: name: Run Checkout Kit workspace setup met?: ./scripts/setup_dev_workspace --check --skip-optional-prompts diff --git a/e2e/.maestro-runner-checksums b/e2e/.maestro-runner-checksums new file mode 100644 index 000000000..c11ac9b2f --- /dev/null +++ b/e2e/.maestro-runner-checksums @@ -0,0 +1,4 @@ +58ffe553995b960606bd47853d205efd65fa91ae7444bd1d976453ecdedde997 maestro-runner-1.1.24-darwin-amd64.tar.gz +2771af3a20aa479a4fd948afab87092daba05c7fed454e8306726a804a288e03 maestro-runner-1.1.24-darwin-arm64.tar.gz +9057c9f1beb87d7b1e92b4d6edadab70e85f39603cd3ed4e690a4ce00ed44563 maestro-runner-1.1.24-linux-amd64.tar.gz +ab357a5d0ac40f79b53f60d862d88fed96e134d314354fb6a898210aa8496ee5 maestro-runner-1.1.24-linux-arm64.tar.gz diff --git a/e2e/.maestro-runner-version b/e2e/.maestro-runner-version new file mode 100644 index 000000000..3fe3e58a8 --- /dev/null +++ b/e2e/.maestro-runner-version @@ -0,0 +1 @@ +1.1.24 diff --git a/e2e/README.md b/e2e/README.md index 160c2b568..cb3f60ac2 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -138,6 +138,20 @@ Count BrowserStack run plan rows: ruby e2e/scripts/e2e_matrix_to_browserstack_run_plan count ``` +## BrowserStack executors + +The Bitrise run stage defaults to the checksum-verified `maestro-runner` pin in +`.maestro-runner-version`. It runs on the Bitrise worker and connects to BrowserStack +real devices through the Appium hub. Set `E2E_BROWSERSTACK_EXECUTOR=hosted-maestro` +to run the previous BrowserStack-hosted Maestro executor for comparison. + +The Appium executor uploads only the app artifact, writes the runner's JSON, JUnit, +HTML, screenshots, hierarchy, logs, and session IDs into the run result artifact, +and preserves the normalized `result.json` contract used by GitHub reporting. It +uses a temporary copy of the E2E workspace to enable `launchApp.newSession`, which +cleans physical iOS state without adding a maestro-runner-only property to the +shared flows consumed by hosted Maestro. + ## Files - `config.yaml` configures Maestro for shared platform behavior and quarantines diff --git a/e2e/bitrise.yml b/e2e/bitrise.yml index 93c45b974..18748307f 100644 --- a/e2e/bitrise.yml +++ b/e2e/bitrise.yml @@ -404,16 +404,33 @@ workflows: envman add --key E2E_BROWSERSTACK_RESULTS_DIR --value "$results_dir" : "${BROWSERSTACK_USERNAME:?BROWSERSTACK_USERNAME is required}" : "${BROWSERSTACK_ACCESS_KEY:?BROWSERSTACK_ACCESS_KEY is required}" - : "${E2E_TESTS_ZIP:?E2E_TESTS_ZIP is required. Check e2e-execute-browserstack-run pull-intermediate-files artifact_sources.}" : "${E2E_BROWSERSTACK_RUN_PLAN_JSON:?E2E_BROWSERSTACK_RUN_PLAN_JSON is required. Check e2e-execute-browserstack-run pull-intermediate-files artifact_sources.}" run_artifact_env="$(ruby -rjson -e 'run_plan = JSON.parse(File.read(ARGV.fetch(0))); puts run_plan.fetch(Integer(ARGV.fetch(1))).fetch("artifact_env")' "$E2E_BROWSERSTACK_RUN_PLAN_JSON" "$run_index")" : "${!run_artifact_env:?${run_artifact_env} is required. Check e2e-execute-browserstack-run pull-intermediate-files artifact_sources.}" - e2e_log "Executing BrowserStack run plan row ${run_index}" - e2e/scripts/execute_browserstack_run \ - --index "$run_index" \ - --run-plan "$E2E_BROWSERSTACK_RUN_PLAN_JSON" \ - --tests-zip "$E2E_TESTS_ZIP" \ - --output-dir "$results_dir" + executor="${E2E_BROWSERSTACK_EXECUTOR:-maestro-runner}" + case "$executor" in + maestro-runner) + scripts/install_maestro_runner + e2e_log "Executing BrowserStack run plan row ${run_index} with maestro-runner" + e2e/scripts/execute_browserstack_runner_run \ + --index "$run_index" \ + --run-plan "$E2E_BROWSERSTACK_RUN_PLAN_JSON" \ + --output-dir "$results_dir" + ;; + hosted-maestro) + : "${E2E_TESTS_ZIP:?E2E_TESTS_ZIP is required for hosted-maestro}" + e2e_log "Executing BrowserStack run plan row ${run_index} with hosted Maestro" + e2e/scripts/execute_browserstack_run \ + --index "$run_index" \ + --run-plan "$E2E_BROWSERSTACK_RUN_PLAN_JSON" \ + --tests-zip "$E2E_TESTS_ZIP" \ + --output-dir "$results_dir" + ;; + *) + e2e_log "Unsupported E2E_BROWSERSTACK_EXECUTOR: $executor" + exit 1 + ;; + esac - deploy-to-bitrise-io@2: is_always_run: true inputs: diff --git a/e2e/lib/browserstack_client.rb b/e2e/lib/browserstack_client.rb index 251a05143..e31ccd958 100644 --- a/e2e/lib/browserstack_client.rb +++ b/e2e/lib/browserstack_client.rb @@ -50,6 +50,16 @@ def upload(path, file_path, custom_id) @client.execute(request) end + def upload_app_automate_app(file_path, custom_id) + upload("/app-automate/upload", file_path, custom_id) + end + + def find_build_by_name(name) + builds = @client.get("/automate/builds.json?limit=100") + match = builds.find { |entry| entry.dig("automation_build", "name") == name } + match&.fetch("automation_build") + end + def start_build(platform, body) @client.post_json("/app-automate/maestro/v2/#{platform}/build", body) end diff --git a/e2e/lib/browserstack_runner_executor.rb b/e2e/lib/browserstack_runner_executor.rb new file mode 100644 index 000000000..ccb9f94fb --- /dev/null +++ b/e2e/lib/browserstack_runner_executor.rb @@ -0,0 +1,256 @@ +# frozen_string_literal: true + +require "fileutils" +require "json" +require "open3" +require "tempfile" +require "time" +require "tmpdir" +require_relative "browserstack_client" +require_relative "browserstack_device_resolver" + +class BrowserStackRunnerExecutor + APPIUM_URL = "https://hub-cloud.browserstack.com/wd/hub" + + def initialize(client:, run:, app_path:, output_dir:, runner_binary:, env: ENV, workspace: File.expand_path("..", __dir__)) + @client = client + @run = run + @app_path = app_path + @output_dir = output_dir + @runner_binary = runner_binary + @env = env + @workspace = workspace + end + + def execute! + validate! + FileUtils.mkdir_p(@output_dir) + + resolved_device = resolve_device + app_url = @client.upload_app_automate_app(@app_path, app_custom_id).fetch("app_url") + build_name = browserstack_build_name + runner_output = File.join(@output_dir, "maestro-runner") + sessions_path = File.join(@output_dir, "appium-sessions.jsonl") + report = nil + runner_passed = false + + with_runner_workspace do |workspace| + Tempfile.create(["checkout-kit-appium-capabilities", ".json"]) do |file| + file.write(JSON.generate(capabilities(@run, resolved_device, app_url, build_name, username, access_key))) + file.flush + command = runner_command(@runner_binary, file.path, runner_output, sessions_path, @run) + runner_passed = run_process(command, File.join(@output_dir, "maestro-runner.log"), workspace) + report = read_report(runner_output) + end + end + + build_id = find_build_id(build_name) + result = normalized_result(@run, resolved_device, report, build_id) + result["passed"] &&= runner_passed + write_result(result) + result.fetch("passed") + rescue StandardError => error + write_result(failure_result(error)) + warn "BrowserStack maestro-runner execution failed: #{error.message}" + false + end + + private + + def validate! + raise "BrowserStack username is required" if username.empty? + raise "BrowserStack access key is required" if access_key.empty? + raise "application artifact does not exist: #{@app_path}" unless File.file?(@app_path) + raise "maestro-runner is not executable: #{@runner_binary}" unless File.executable?(@runner_binary) + end + + def username + @env.fetch("BROWSERSTACK_USERNAME", "") + end + + def access_key + @env.fetch("BROWSERSTACK_ACCESS_KEY", "") + end + + def resolve_device + devices = @client.list_devices + limits = @client.list_device_tier_limits + BrowserStackDeviceResolver.new(devices, limits).resolve(@run.fetch("device_selector")) + end + + def app_custom_id + commit = @env["GIT_COMMIT"] || @env["BITRISE_GIT_COMMIT"] || "local" + "checkout-kit-#{@run.fetch("target")}-#{@run.fetch("platform")}-#{commit}".gsub(/[^a-zA-Z0-9_.-]/, "-") + end + + def browserstack_build_name + identifier = @env["BITRISE_BUILD_NUMBER"] || @env["GITHUB_RUN_ID"] || @env["GIT_COMMIT"] || Time.now.utc.strftime("%Y%m%d%H%M%S") + "checkout-kit maestro-runner #{identifier} #{@run.fetch("id")}" + end + + def find_build_id(build_name) + @client.find_build_by_name(build_name)&.fetch("hashed_id", nil) + rescue StandardError => error + warn "Unable to resolve BrowserStack build #{build_name}: #{error.message}" + nil + end + + def capabilities(run, resolved_device, app_url, build_name, browserstack_username, browserstack_access_key) + platform = run.fetch("platform") + result = { + "platformName" => platform == "ios" ? "iOS" : "Android", + "appium:automationName" => platform == "ios" ? "XCUITest" : "UiAutomator2", + "appium:app" => app_url, + "appium:deviceName" => resolved_device.fetch("resolved_device"), + "appium:platformVersion" => resolved_device.fetch("resolved_os_version"), + "appium:noReset" => false, + "appium:fullReset" => false, + "appium:newCommandTimeout" => 300, + "bstack:options" => { + "userName" => browserstack_username, + "accessKey" => browserstack_access_key, + "projectName" => "Checkout Kit E2E", + "buildName" => build_name, + "sessionName" => run.fetch("id"), + "appiumVersion" => "latest", + "language" => "en", + "locale" => "US", + "debug" => true, + "video" => true, + "deviceLogs" => true, + "networkLogs" => true, + "idleTimeout" => 300 + } + } + + if platform == "ios" + result["appium:bundleId"] = run.fetch("app_id") + else + result["appium:appPackage"] = run.fetch("app_id") + result["appium:autoGrantPermissions"] = true + end + + result + end + + def runner_command(binary, caps_path, runner_output, sessions_path, run) + command = [ + binary, + "--driver", "appium", + "--platform", run.fetch("platform"), + "--appium-url", APPIUM_URL, + "--caps", caps_path, + "--appium-session-file", sessions_path, + "test", + "--output", runner_output, + "--flatten", + "--artifacts", "always" + ] + + comma_separated(run["include_tags"]).each { |tag| command.concat(["--include-tags", tag]) } + comma_separated(run["exclude_tags"]).each { |tag| command.concat(["--exclude-tags", tag]) } + flow_environment(run).each { |key, value| command.concat(["--env", "#{key}=#{value}"]) } + command.concat(test_paths(run)) + end + + def test_paths(run) + paths = ["tests/shared"] + target_path = "tests/#{run.fetch("target")}" + paths << target_path if @workspace && File.directory?(File.join(@workspace, target_path)) + paths + end + + def comma_separated(value) + values = value.is_a?(Array) ? value : value.to_s.split(",") + values.map(&:to_s).map(&:strip).reject(&:empty?) + end + + def flow_environment(run) + { + "E2E_APP_ID" => run.fetch("app_id"), + "E2E_READY_MARKER" => run.fetch("ready_marker"), + "E2E_CONTROL_LINK" => run.fetch("control_link") + }.merge(run.fetch("env", {})) + end + + def with_runner_workspace + Dir.mktmpdir("checkout-kit-maestro-runner") do |workspace| + FileUtils.cp_r(File.join(@workspace, "."), workspace) + launch_path = File.join(workspace, "flows/app/launch.yaml") + launch_flow = File.read(launch_path) + updated_flow = launch_flow.sub(/^(\s*)clearState: true$/, "\\1clearState: true\n\\1newSession: true") + raise "could not enable Appium newSession in #{launch_path}" if updated_flow == launch_flow + + File.write(launch_path, updated_flow) + yield workspace + end + end + + def run_process(command, log_path, workspace) + success = false + File.open(log_path, "w") do |log| + Open3.popen2e(*command, chdir: workspace) do |_stdin, output, wait_thread| + output.each do |line| + $stdout.write(line) + log.write(line) + end + success = wait_thread.value.success? + end + end + success + end + + def read_report(runner_output) + path = File.join(runner_output, "report.json") + raise "maestro-runner did not produce #{path}" unless File.file?(path) + + JSON.parse(File.read(path)) + end + + def normalized_result(run, resolved_device, report, build_id) + failed_tests = report.fetch("flows", []).filter_map do |flow| + next unless flow["status"] == "failed" + + { + "name" => flow.fetch("name", "unknown"), + "source_file" => flow["sourceFile"], + "duration_ms" => flow["duration"], + "error" => flow["error"] + }.compact + end + + { + "id" => run.fetch("id"), + "target" => run.fetch("target"), + "application_id" => run.fetch("application_id"), + "platform" => run.fetch("platform"), + "os_version_tag" => run.fetch("os_version_tag"), + "resolved_device" => resolved_device.fetch("resolved_device"), + "resolved_os_version" => resolved_device.fetch("resolved_os_version"), + "passed" => report["status"] == "passed", + "duration_ms" => report.fetch("flows", []).sum { |flow| flow.fetch("duration", 0).to_i }, + "summary" => report.fetch("summary", {}), + "failed_tests" => failed_tests, + "build_id" => build_id, + "executor" => "maestro-runner" + } + end + + def failure_result(error) + { + "id" => @run["id"], + "target" => @run["target"], + "application_id" => @run["application_id"], + "platform" => @run["platform"], + "os_version_tag" => @run["os_version_tag"], + "passed" => false, + "failed_tests" => [{"name" => "maestro-runner setup", "error" => error.message}], + "executor" => "maestro-runner" + }.compact + end + + def write_result(result) + FileUtils.mkdir_p(@output_dir) + File.write(File.join(@output_dir, "result.json"), JSON.pretty_generate(result)) + end +end diff --git a/e2e/scripts/execute_browserstack_runner_run b/e2e/scripts/execute_browserstack_runner_run new file mode 100755 index 000000000..fe2f06930 --- /dev/null +++ b/e2e/scripts/execute_browserstack_runner_run @@ -0,0 +1,41 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "json" +require "optparse" +require_relative "../lib/browserstack_client" +require_relative "../lib/browserstack_runner_executor" + +options = { + run_plan_path: ENV["E2E_BROWSERSTACK_RUN_PLAN_JSON"] || "browserstack-run-plan.json", + run_index: ENV["BITRISE_IO_PARALLEL_INDEX"]&.to_i, + output_dir: ENV["E2E_BROWSERSTACK_RESULTS_DIR"] || File.join(Dir.pwd, "e2e-results") +} + +OptionParser.new do |parser| + parser.on("--run-plan PATH") { |path| options[:run_plan_path] = path } + parser.on("--index INDEX", Integer) { |index| options[:run_index] = index } + parser.on("--output-dir PATH") { |path| options[:output_dir] = path } +end.parse! + +raise ArgumentError, "run index is required" if options[:run_index].nil? + +run_plan = JSON.parse(File.read(options.fetch(:run_plan_path))) +run = run_plan.fetch(options.fetch(:run_index)) +app_path = ENV.fetch(run.fetch("artifact_env")) +runner_binary = `#{File.expand_path("maestro_runner_bin", __dir__)}`.strip +raise "could not resolve maestro-runner" unless $?.success? + +client = BrowserStackClient.new( + username: ENV.fetch("BROWSERSTACK_USERNAME"), + access_key: ENV.fetch("BROWSERSTACK_ACCESS_KEY"), + retries: ENV.fetch("E2E_BROWSERSTACK_API_RETRIES", "1").to_i +) + +BrowserStackRunnerExecutor.new( + client: client, + run: run, + app_path: app_path, + output_dir: options.fetch(:output_dir), + runner_binary: runner_binary +).execute! diff --git a/e2e/scripts/maestro_runner_bin b/e2e/scripts/maestro_runner_bin new file mode 100755 index 000000000..631d1e7de --- /dev/null +++ b/e2e/scripts/maestro_runner_bin @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail + +E2E_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +VERSIONS_ROOT="${MAESTRO_RUNNER_VERSIONS_ROOT:-$HOME/.maestro-runner-versions}" +version="$(tr -d '[:space:]' <"$E2E_ROOT/.maestro-runner-version")" +binary="$VERSIONS_ROOT/$version/maestro-runner" + +if [ ! -x "$binary" ]; then + echo "maestro_runner_bin: maestro-runner $version is not installed at $binary." >&2 + echo " run: scripts/install_maestro_runner" >&2 + exit 1 +fi + +printf '%s\n' "$binary" diff --git a/e2e/test/browserstack_client_test.rb b/e2e/test/browserstack_client_test.rb index ff14f0362..c3c92f0d7 100644 --- a/e2e/test/browserstack_client_test.rb +++ b/e2e/test/browserstack_client_test.rb @@ -4,6 +4,13 @@ require_relative "../lib/browserstack_client" class BrowserStackClientTest < Minitest::Test + FakeHttpClient = Struct.new(:response, :path) do + def get(request_path) + self.path = request_path + response + end + end + def test_build_url_appends_build_id assert_equal "https://app-automate.browserstack.com/dashboard/v2/builds/abc123", BrowserStackClient.build_url("abc123") end @@ -12,4 +19,31 @@ def test_build_url_without_build_id_returns_dashboard_base assert_equal "https://app-automate.browserstack.com/dashboard/v2/builds", BrowserStackClient.build_url(nil) assert_equal "https://app-automate.browserstack.com/dashboard/v2/builds", BrowserStackClient.build_url("") end + + def test_standard_app_upload_uses_the_app_automate_endpoint + client = BrowserStackClient.allocate + client.define_singleton_method(:upload) do |path, file_path, custom_id| + {"path" => path, "file_path" => file_path, "custom_id" => custom_id} + end + + response = client.upload_app_automate_app("sample.ipa", "checkout-kit-ios-abc") + + assert_equal "/app-automate/upload", response.fetch("path") + assert_equal "sample.ipa", response.fetch("file_path") + assert_equal "checkout-kit-ios-abc", response.fetch("custom_id") + end + + def test_find_build_by_name_returns_the_standard_app_automate_build + http = FakeHttpClient.new([ + {"automation_build" => {"name" => "other", "hashed_id" => "first"}}, + {"automation_build" => {"name" => "checkout-kit maestro-runner 42", "hashed_id" => "wanted"}} + ]) + client = BrowserStackClient.allocate + client.instance_variable_set(:@client, http) + + build = client.find_build_by_name("checkout-kit maestro-runner 42") + + assert_equal "wanted", build.fetch("hashed_id") + assert_equal "/automate/builds.json?limit=100", http.path + end end diff --git a/e2e/test/browserstack_runner_executor_test.rb b/e2e/test/browserstack_runner_executor_test.rb new file mode 100644 index 000000000..eb3cc0f3f --- /dev/null +++ b/e2e/test/browserstack_runner_executor_test.rb @@ -0,0 +1,174 @@ +# frozen_string_literal: true + +require "minitest/autorun" +require "tmpdir" +require_relative "../lib/browserstack_runner_executor" + +class BrowserStackRunnerExecutorTest < Minitest::Test + def setup + @executor = BrowserStackRunnerExecutor.allocate + end + + def test_android_capabilities_install_the_uploaded_app_on_the_resolved_device + capabilities = @executor.send( + :capabilities, + { + "id" => "kotlin-android-latest", + "application_id" => "kotlin-android", + "app_id" => "com.shopify.checkoutkit", + "platform" => "android" + }, + { + "resolved_device" => "Google Pixel 9", + "resolved_os_version" => "15.0" + }, + "bs://uploaded-app", + "checkout-kit maestro-runner 123", + "user", + "key" + ) + + assert_equal "Android", capabilities.fetch("platformName") + assert_equal "UiAutomator2", capabilities.fetch("appium:automationName") + assert_equal "bs://uploaded-app", capabilities.fetch("appium:app") + assert_equal "Google Pixel 9", capabilities.fetch("appium:deviceName") + assert_equal "15.0", capabilities.fetch("appium:platformVersion") + assert_equal "com.shopify.checkoutkit", capabilities.fetch("appium:appPackage") + assert_equal false, capabilities.fetch("appium:noReset") + assert_equal true, capabilities.fetch("appium:autoGrantPermissions") + + browserstack = capabilities.fetch("bstack:options") + assert_equal "user", browserstack.fetch("userName") + assert_equal "key", browserstack.fetch("accessKey") + assert_equal "checkout-kit maestro-runner 123", browserstack.fetch("buildName") + assert_equal "kotlin-android-latest", browserstack.fetch("sessionName") + assert_equal "en", browserstack.fetch("language") + assert_equal "US", browserstack.fetch("locale") + end + + def test_ios_capabilities_use_xcuitest_and_the_bundle_identifier + capabilities = @executor.send( + :capabilities, + { + "id" => "swift-ios-latest", + "application_id" => "swift-ios", + "app_id" => "com.shopify.checkoutkit", + "platform" => "ios" + }, + { + "resolved_device" => "iPhone 16", + "resolved_os_version" => "18" + }, + "bs://uploaded-app", + "checkout-kit maestro-runner abc", + "user", + "key" + ) + + assert_equal "iOS", capabilities.fetch("platformName") + assert_equal "XCUITest", capabilities.fetch("appium:automationName") + assert_equal "com.shopify.checkoutkit", capabilities.fetch("appium:bundleId") + refute capabilities.key?("appium:autoGrantPermissions") + end + + def test_command_translates_tags_and_environment_to_repeated_runner_flags + command = @executor.send( + :runner_command, + "/tmp/maestro-runner", + "/tmp/caps.json", + "/tmp/results", + "/tmp/sessions.jsonl", + { + "platform" => "ios", + "target" => "swift", + "app_id" => "com.shopify.checkoutkit", + "ready_marker" => "sample-ready", + "control_link" => "com.shopify.checkoutkit://e2e", + "include_tags" => ["launch", "checkout"], + "exclude_tags" => ["android", "local"], + "env" => {"CART" => "synthetic-cart", "EMPTY" => ""} + } + ) + + assert_equal [ + "/tmp/maestro-runner", + "--driver", "appium", + "--platform", "ios", + "--appium-url", BrowserStackRunnerExecutor::APPIUM_URL, + "--caps", "/tmp/caps.json", + "--appium-session-file", "/tmp/sessions.jsonl", + "test", + "--output", "/tmp/results", + "--flatten", + "--artifacts", "always", + "--include-tags", "launch", + "--include-tags", "checkout", + "--exclude-tags", "android", + "--exclude-tags", "local", + "--env", "E2E_APP_ID=com.shopify.checkoutkit", + "--env", "E2E_READY_MARKER=sample-ready", + "--env", "E2E_CONTROL_LINK=com.shopify.checkoutkit://e2e", + "--env", "CART=synthetic-cart", + "--env", "EMPTY=", + "tests/shared" + ], command + end + + def test_runner_workspace_enables_a_fresh_appium_session_without_changing_the_shared_flow + Dir.mktmpdir do |source| + launch_path = File.join(source, "flows/app/launch.yaml") + FileUtils.mkdir_p(File.dirname(launch_path)) + File.write(launch_path, "- launchApp:\n clearState: true\n arguments: {}\n") + @executor.instance_variable_set(:@workspace, source) + + @executor.send(:with_runner_workspace) do |workspace| + copied_launch = File.read(File.join(workspace, "flows/app/launch.yaml")) + assert_includes copied_launch, "clearState: true\n newSession: true" + end + + refute_includes File.read(launch_path), "newSession" + end + end + + def test_normalized_result_uses_runner_report_for_failures_and_timing + result = @executor.send( + :normalized_result, + { + "id" => "react-native-ios-latest", + "target" => "react-native-ios", + "application_id" => "com.shopify.checkoutkit", + "platform" => "ios", + "os_version_tag" => "latest" + }, + { + "resolved_device" => "iPhone 16", + "resolved_os_version" => "18" + }, + { + "status" => "failed", + "startTime" => "2026-03-10T10:00:00Z", + "endTime" => "2026-03-10T10:01:02.500Z", + "summary" => {"total" => 2, "passed" => 1, "failed" => 1}, + "flows" => [ + {"name" => "Launch smoke", "status" => "passed", "duration" => 12_000}, + {"name" => "Guest checkout", "sourceFile" => "tests/shared/checkout-guest.yaml", "status" => "failed", "duration" => 50_500, "error" => "Card field timed out"} + ] + }, + "build-hash" + ) + + assert_equal false, result.fetch("passed") + assert_equal 62_500, result.fetch("duration_ms") + assert_equal({"total" => 2, "passed" => 1, "failed" => 1}, result.fetch("summary")) + assert_equal [ + { + "name" => "Guest checkout", + "source_file" => "tests/shared/checkout-guest.yaml", + "duration_ms" => 50_500, + "error" => "Card field timed out" + } + ], result.fetch("failed_tests") + assert_equal "build-hash", result.fetch("build_id") + assert_equal "maestro-runner", result.fetch("executor") + end +end diff --git a/e2e/test/maestro_runner_installation_test.rb b/e2e/test/maestro_runner_installation_test.rb new file mode 100644 index 000000000..2e8429429 --- /dev/null +++ b/e2e/test/maestro_runner_installation_test.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +require "minitest/autorun" +require "open3" +require "tmpdir" + +class MaestroRunnerInstallationTest < Minitest::Test + REPO_ROOT = File.expand_path("../..", __dir__) + VERSION_FILE = File.join(REPO_ROOT, "e2e/.maestro-runner-version") + CHECKSUMS_FILE = File.join(REPO_ROOT, "e2e/.maestro-runner-checksums") + INSTALLER = File.join(REPO_ROOT, "scripts/install_maestro_runner") + RESOLVER = File.join(REPO_ROOT, "e2e/scripts/maestro_runner_bin") + + def test_release_is_pinned_with_checksums_for_every_supported_worker + version = File.read(VERSION_FILE).strip + checksums = File.readlines(CHECKSUMS_FILE, chomp: true).to_h { |line| line.split(" ", 2).reverse } + + assert_equal "1.1.24", version + %w[darwin-amd64 darwin-arm64 linux-amd64 linux-arm64].each do |platform| + asset = "maestro-runner-#{version}-#{platform}.tar.gz" + assert_match(/\A[0-9a-f]{64}\z/, checksums.fetch(asset)) + end + end + + def test_resolver_returns_the_pinned_executable + Dir.mktmpdir do |versions_root| + version = File.read(VERSION_FILE).strip + binary = File.join(versions_root, version, "maestro-runner") + FileUtils.mkdir_p(File.dirname(binary)) + File.write(binary, "#!/bin/sh\nexit 0\n") + File.chmod(0o755, binary) + + stdout, stderr, status = Open3.capture3({"MAESTRO_RUNNER_VERSIONS_ROOT" => versions_root}, RESOLVER) + + assert status.success?, stderr + assert_equal "#{binary}\n", stdout + end + end +end diff --git a/scripts/install_maestro_runner b/scripts/install_maestro_runner new file mode 100755 index 000000000..92f264e21 --- /dev/null +++ b/scripts/install_maestro_runner @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +VERSIONS_ROOT="${MAESTRO_RUNNER_VERSIONS_ROOT:-$HOME/.maestro-runner-versions}" +version="$(tr -d '[:space:]' <"$REPO_ROOT/e2e/.maestro-runner-version")" +destination="$VERSIONS_ROOT/$version" +binary="$destination/maestro-runner" +mode="install" + +if [ "${1:-}" = "--check" ]; then + mode="check" +elif [ "$#" -ne 0 ]; then + echo "Usage: scripts/install_maestro_runner [--check]" >&2 + exit 1 +fi + +installed_version() { + if [ ! -x "$binary" ]; then + return 1 + fi + "$binary" --version 2>/dev/null | head -1 | awk '{print $2}' +} + +if [ "$mode" = "check" ]; then + found="$(installed_version || true)" + if [ "$found" != "$version" ]; then + echo "install_maestro_runner: expected $version at $binary, found ${found:-nothing}." >&2 + exit 1 + fi + exit 0 +fi + +if [ "$(installed_version || true)" = "$version" ]; then + echo "maestro-runner $version is installed at $binary" + exit 0 +fi + +case "$(uname -s)" in + Darwin) os="darwin" ;; + Linux) os="linux" ;; + *) echo "install_maestro_runner: unsupported OS $(uname -s)" >&2; exit 1 ;; +esac + +case "$(uname -m)" in + x86_64|amd64) arch="amd64" ;; + arm64|aarch64) arch="arm64" ;; + *) echo "install_maestro_runner: unsupported architecture $(uname -m)" >&2; exit 1 ;; +esac + +asset="maestro-runner-$version-$os-$arch.tar.gz" +expected_checksum="$(awk -v asset="$asset" '$2 == asset { print $1 }' "$REPO_ROOT/e2e/.maestro-runner-checksums")" +if [ -z "$expected_checksum" ]; then + echo "install_maestro_runner: no checksum pinned for $asset" >&2 + exit 1 +fi + +scratch="$(mktemp -d "${TMPDIR:-/tmp}/checkout-kit-maestro-runner.XXXXXX")" +trap 'rm -rf "$scratch"' EXIT +archive="$scratch/$asset" +curl -fsSL "https://github.com/devicelab-dev/maestro-runner/releases/download/v$version/$asset" -o "$archive" +if command -v shasum >/dev/null 2>&1; then + actual_checksum="$(shasum -a 256 "$archive" | awk '{print $1}')" +else + actual_checksum="$(sha256sum "$archive" | awk '{print $1}')" +fi +if [ "$actual_checksum" != "$expected_checksum" ]; then + echo "install_maestro_runner: checksum mismatch for $asset" >&2 + exit 1 +fi + +mkdir -p "$destination" +tar -xzf "$archive" -C "$scratch" +extracted="$(find "$scratch" -type f -name maestro-runner -perm -111 | head -1)" +if [ -z "$extracted" ]; then + echo "install_maestro_runner: $asset did not contain an executable" >&2 + exit 1 +fi +install -m 755 "$extracted" "$binary" + +if [ "$(installed_version || true)" != "$version" ]; then + echo "install_maestro_runner: $binary does not report $version" >&2 + exit 1 +fi + +echo "maestro-runner $version is installed at $binary" From 9179325b6126d92a8adf5e9b16ec3a3b1bd28600 Mon Sep 17 00:00:00 2001 From: Kieran Osgood Date: Thu, 20 Aug 2026 19:31:20 +0100 Subject: [PATCH 2/5] Fix BrowserStack Appium capabilities --- e2e/lib/browserstack_runner_executor.rb | 23 +++++++++++++++---- e2e/test/browserstack_runner_executor_test.rb | 15 ++++++++++-- 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/e2e/lib/browserstack_runner_executor.rb b/e2e/lib/browserstack_runner_executor.rb index ccb9f94fb..546adc392 100644 --- a/e2e/lib/browserstack_runner_executor.rb +++ b/e2e/lib/browserstack_runner_executor.rb @@ -53,6 +53,8 @@ def execute! write_result(failure_result(error)) warn "BrowserStack maestro-runner execution failed: #{error.message}" false + ensure + redact_artifacts end private @@ -113,8 +115,6 @@ def capabilities(run, resolved_device, app_url, build_name, browserstack_usernam "buildName" => build_name, "sessionName" => run.fetch("id"), "appiumVersion" => "latest", - "language" => "en", - "locale" => "US", "debug" => true, "video" => true, "deviceLogs" => true, @@ -191,8 +191,9 @@ def run_process(command, log_path, workspace) File.open(log_path, "w") do |log| Open3.popen2e(*command, chdir: workspace) do |_stdin, output, wait_thread| output.each do |line| - $stdout.write(line) - log.write(line) + safe_line = redact(line) + $stdout.write(safe_line) + log.write(safe_line) end success = wait_thread.value.success? end @@ -200,6 +201,20 @@ def run_process(command, log_path, workspace) success end + def redact(value) + [username, access_key].reject(&:empty?).reduce(value) { |text, secret| text.gsub(secret, "[REDACTED]") } + end + + def redact_artifacts + return unless @output_dir && File.directory?(@output_dir) + + Dir.glob(File.join(@output_dir, "**", "*.{html,json,jsonl,log,txt,xml}")).each do |path| + contents = File.binread(path) + redacted = redact(contents) + File.binwrite(path, redacted) if redacted != contents + end + end + def read_report(runner_output) path = File.join(runner_output, "report.json") raise "maestro-runner did not produce #{path}" unless File.file?(path) diff --git a/e2e/test/browserstack_runner_executor_test.rb b/e2e/test/browserstack_runner_executor_test.rb index eb3cc0f3f..a7d0b9cc3 100644 --- a/e2e/test/browserstack_runner_executor_test.rb +++ b/e2e/test/browserstack_runner_executor_test.rb @@ -42,8 +42,8 @@ def test_android_capabilities_install_the_uploaded_app_on_the_resolved_device assert_equal "key", browserstack.fetch("accessKey") assert_equal "checkout-kit maestro-runner 123", browserstack.fetch("buildName") assert_equal "kotlin-android-latest", browserstack.fetch("sessionName") - assert_equal "en", browserstack.fetch("language") - assert_equal "US", browserstack.fetch("locale") + refute browserstack.key?("language") + refute browserstack.key?("locale") end def test_ios_capabilities_use_xcuitest_and_the_bundle_identifier @@ -114,6 +114,17 @@ def test_command_translates_tags_and_environment_to_repeated_runner_flags ], command end + def test_redaction_removes_browserstack_credentials_from_runner_output + @executor.instance_variable_set(:@env, { + "BROWSERSTACK_USERNAME" => "browserstack-user", + "BROWSERSTACK_ACCESS_KEY" => "browserstack-key" + }) + + redacted = @executor.send(:redact, "user=browserstack-user key=browserstack-key") + + assert_equal "user=[REDACTED] key=[REDACTED]", redacted + end + def test_runner_workspace_enables_a_fresh_appium_session_without_changing_the_shared_flow Dir.mktmpdir do |source| launch_path = File.join(source, "flows/app/launch.yaml") From babac0035ef4b287ee1e3f1bdd30e5563401b70f Mon Sep 17 00:00:00 2001 From: Kieran Osgood Date: Thu, 20 Aug 2026 19:40:02 +0100 Subject: [PATCH 3/5] Use BrowserStack default Appium version --- e2e/lib/browserstack_runner_executor.rb | 1 - e2e/test/browserstack_runner_executor_test.rb | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/lib/browserstack_runner_executor.rb b/e2e/lib/browserstack_runner_executor.rb index 546adc392..61cbe1581 100644 --- a/e2e/lib/browserstack_runner_executor.rb +++ b/e2e/lib/browserstack_runner_executor.rb @@ -114,7 +114,6 @@ def capabilities(run, resolved_device, app_url, build_name, browserstack_usernam "projectName" => "Checkout Kit E2E", "buildName" => build_name, "sessionName" => run.fetch("id"), - "appiumVersion" => "latest", "debug" => true, "video" => true, "deviceLogs" => true, diff --git a/e2e/test/browserstack_runner_executor_test.rb b/e2e/test/browserstack_runner_executor_test.rb index a7d0b9cc3..215bb79ca 100644 --- a/e2e/test/browserstack_runner_executor_test.rb +++ b/e2e/test/browserstack_runner_executor_test.rb @@ -42,6 +42,7 @@ def test_android_capabilities_install_the_uploaded_app_on_the_resolved_device assert_equal "key", browserstack.fetch("accessKey") assert_equal "checkout-kit maestro-runner 123", browserstack.fetch("buildName") assert_equal "kotlin-android-latest", browserstack.fetch("sessionName") + refute browserstack.key?("appiumVersion") refute browserstack.key?("language") refute browserstack.key?("locale") end From 79e3388d0e755782fcc6d21cc3f94b90df8245f3 Mon Sep 17 00:00:00 2001 From: Kieran Osgood Date: Thu, 20 Aug 2026 20:04:55 +0100 Subject: [PATCH 4/5] Adapt launch state for BrowserStack iOS --- e2e/README.md | 7 ++++--- e2e/lib/browserstack_runner_executor.rb | 9 +++++++-- e2e/test/browserstack_runner_executor_test.rb | 19 +++++++++++++++++++ 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/e2e/README.md b/e2e/README.md index cb3f60ac2..29ac38d79 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -148,9 +148,10 @@ to run the previous BrowserStack-hosted Maestro executor for comparison. The Appium executor uploads only the app artifact, writes the runner's JSON, JUnit, HTML, screenshots, hierarchy, logs, and session IDs into the run result artifact, and preserves the normalized `result.json` contract used by GitHub reporting. It -uses a temporary copy of the E2E workspace to enable `launchApp.newSession`, which -cleans physical iOS state without adding a maestro-runner-only property to the -shared flows consumed by hosted Maestro. +uses a temporary copy of the E2E workspace to enable `launchApp.newSession` on +Android. On iOS it relies on BrowserStack's clean physical-device session and skips +the simulator-only `clearState` command that BrowserStack cannot execute. The +shared flows consumed by hosted Maestro remain unchanged. ## Files diff --git a/e2e/lib/browserstack_runner_executor.rb b/e2e/lib/browserstack_runner_executor.rb index 61cbe1581..000571822 100644 --- a/e2e/lib/browserstack_runner_executor.rb +++ b/e2e/lib/browserstack_runner_executor.rb @@ -177,8 +177,13 @@ def with_runner_workspace FileUtils.cp_r(File.join(@workspace, "."), workspace) launch_path = File.join(workspace, "flows/app/launch.yaml") launch_flow = File.read(launch_path) - updated_flow = launch_flow.sub(/^(\s*)clearState: true$/, "\\1clearState: true\n\\1newSession: true") - raise "could not enable Appium newSession in #{launch_path}" if updated_flow == launch_flow + replacement = if @run.fetch("platform") == "ios" + "\\1clearState: false" + else + "\\1clearState: true\n\\1newSession: true" + end + updated_flow = launch_flow.sub(/^(\s*)clearState: true$/, replacement) + raise "could not adapt Appium launch in #{launch_path}" if updated_flow == launch_flow File.write(launch_path, updated_flow) yield workspace diff --git a/e2e/test/browserstack_runner_executor_test.rb b/e2e/test/browserstack_runner_executor_test.rb index 215bb79ca..3f345e034 100644 --- a/e2e/test/browserstack_runner_executor_test.rb +++ b/e2e/test/browserstack_runner_executor_test.rb @@ -132,6 +132,7 @@ def test_runner_workspace_enables_a_fresh_appium_session_without_changing_the_sh FileUtils.mkdir_p(File.dirname(launch_path)) File.write(launch_path, "- launchApp:\n clearState: true\n arguments: {}\n") @executor.instance_variable_set(:@workspace, source) + @executor.instance_variable_set(:@run, {"platform" => "android"}) @executor.send(:with_runner_workspace) do |workspace| copied_launch = File.read(File.join(workspace, "flows/app/launch.yaml")) @@ -142,6 +143,24 @@ def test_runner_workspace_enables_a_fresh_appium_session_without_changing_the_sh end end + def test_ios_runner_workspace_skips_the_simulator_only_clear_state_command + Dir.mktmpdir do |source| + launch_path = File.join(source, "flows/app/launch.yaml") + FileUtils.mkdir_p(File.dirname(launch_path)) + File.write(launch_path, "- launchApp:\n clearState: true\n arguments: {}\n") + @executor.instance_variable_set(:@workspace, source) + @executor.instance_variable_set(:@run, {"platform" => "ios"}) + + @executor.send(:with_runner_workspace) do |workspace| + copied_launch = File.read(File.join(workspace, "flows/app/launch.yaml")) + assert_includes copied_launch, "clearState: false" + refute_includes copied_launch, "newSession" + end + + assert_includes File.read(launch_path), "clearState: true" + end + end + def test_normalized_result_uses_runner_report_for_failures_and_timing result = @executor.send( :normalized_result, From 271adbe127f1a39cedc27ed5b0ceb5e66b0a48d4 Mon Sep 17 00:00:00 2001 From: Kieran Osgood Date: Thu, 20 Aug 2026 20:54:55 +0100 Subject: [PATCH 5/5] Keep hosted Maestro as the E2E default --- e2e/README.md | 8 ++++---- e2e/bitrise.yml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/e2e/README.md b/e2e/README.md index 29ac38d79..20aa612b2 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -140,10 +140,10 @@ ruby e2e/scripts/e2e_matrix_to_browserstack_run_plan count ## BrowserStack executors -The Bitrise run stage defaults to the checksum-verified `maestro-runner` pin in -`.maestro-runner-version`. It runs on the Bitrise worker and connects to BrowserStack -real devices through the Appium hub. Set `E2E_BROWSERSTACK_EXECUTOR=hosted-maestro` -to run the previous BrowserStack-hosted Maestro executor for comparison. +The Bitrise run stage defaults to BrowserStack-hosted Maestro. Set +`E2E_BROWSERSTACK_EXECUTOR=maestro-runner` to use the checksum-verified pin in +`.maestro-runner-version`; it runs on the Bitrise worker and connects to BrowserStack +real devices through the Appium hub. The Appium executor uploads only the app artifact, writes the runner's JSON, JUnit, HTML, screenshots, hierarchy, logs, and session IDs into the run result artifact, diff --git a/e2e/bitrise.yml b/e2e/bitrise.yml index 18748307f..83b248d5e 100644 --- a/e2e/bitrise.yml +++ b/e2e/bitrise.yml @@ -407,7 +407,7 @@ workflows: : "${E2E_BROWSERSTACK_RUN_PLAN_JSON:?E2E_BROWSERSTACK_RUN_PLAN_JSON is required. Check e2e-execute-browserstack-run pull-intermediate-files artifact_sources.}" run_artifact_env="$(ruby -rjson -e 'run_plan = JSON.parse(File.read(ARGV.fetch(0))); puts run_plan.fetch(Integer(ARGV.fetch(1))).fetch("artifact_env")' "$E2E_BROWSERSTACK_RUN_PLAN_JSON" "$run_index")" : "${!run_artifact_env:?${run_artifact_env} is required. Check e2e-execute-browserstack-run pull-intermediate-files artifact_sources.}" - executor="${E2E_BROWSERSTACK_EXECUTOR:-maestro-runner}" + executor="${E2E_BROWSERSTACK_EXECUTOR:-hosted-maestro}" case "$executor" in maestro-runner) scripts/install_maestro_runner