DO NOT MERGE: test(e2e): preflight native preload logs on BrowserStack

e2e/bitrise.yml

@@ -31,7 +31,7 @@ tools:
 
 trigger_map:
   - pull_request_source_branch: "*"
-    draft_pull_request_enabled: false
+    draft_pull_request_enabled: true
     changed_files:
       # Bitrise evaluates this with Go's RE2 engine. Use only basic syntax
       # (anchors, groups, alternation, character classes, quantifiers). Do not
@@ -197,7 +197,7 @@ workflows:
                 else
                   git diff --name-only "${BITRISE_GIT_BRANCH_DEST:-origin/main}...HEAD" > "$changed_files"
                 fi
-                ruby e2e/scripts/e2e_matrix_to_browserstack_run_plan validate --changed-files-file "$changed_files"
+                ruby e2e/scripts/e2e_matrix_to_browserstack_run_plan validate --config e2e/config/preload-log-preflight-matrix.yml --changed-files-file "$changed_files"
                 # Guard against config-vs-matrix drift: Bitrise resolves the pipeline graph from
                 # the branch-head bitrise.yml (before the merge checkout), while the matrix here is
                 # generated from the merged tree. A stale branch can select a target whose build
@@ -208,12 +208,12 @@ workflows:
                 if [ -z "${BITRISE_PULL_REQUEST:-}" ] || [ -z "${BITRISE_GIT_COMMIT:-}" ] || ! git show "${BITRISE_GIT_COMMIT}:e2e/bitrise.yml" > "$branch_config" 2>/dev/null; then
                   cp e2e/bitrise.yml "$branch_config"
                 fi
-                ruby e2e/scripts/e2e_matrix_to_browserstack_run_plan assert-pipeline-coverage --pipeline-config "$branch_config" --changed-files-file "$changed_files"
+                ruby e2e/scripts/e2e_matrix_to_browserstack_run_plan assert-pipeline-coverage --config e2e/config/preload-log-preflight-matrix.yml --pipeline-config "$branch_config" --changed-files-file "$changed_files"
                 e2e_log "Producing BrowserStack run plan"
-                ruby e2e/scripts/e2e_matrix_to_browserstack_run_plan expand --changed-files-file "$changed_files" > "$BITRISE_DEPLOY_DIR/browserstack-run-plan.json"
+                ruby e2e/scripts/e2e_matrix_to_browserstack_run_plan expand --config e2e/config/preload-log-preflight-matrix.yml --changed-files-file "$changed_files" > "$BITRISE_DEPLOY_DIR/browserstack-run-plan.json"
                 e2e_log "Publishing BrowserStack run plan environment"
                 bitrise_env_file="$(e2e_deploy_dir)/bitrise-env.txt"
-                ruby e2e/scripts/e2e_matrix_to_browserstack_run_plan bitrise-env --changed-files-file "$changed_files" > "$bitrise_env_file"
+                ruby e2e/scripts/e2e_matrix_to_browserstack_run_plan bitrise-env --config e2e/config/preload-log-preflight-matrix.yml --changed-files-file "$changed_files" > "$bitrise_env_file"
                 while IFS='=' read -r key value; do
                   envman add --key "$key" --value "$value"
                 done < "$bitrise_env_file"

e2e/config/preload-log-preflight-matrix.yml

@@ -0,0 +1,33 @@
+version: 1
+changed_file_filters: .ci/changed-file-filters.yml
+applications:
+  - id: kotlin-android
+    target: kotlin
+    platform: android
+    app_id: com.shopify.checkoutkit.androiddemo
+    artifact_env: E2E_KOTLIN_ANDROID_APP_PATH
+    ready_marker: checkout-kit-sample-ready
+    changed_files_filters:
+      - android
+      - protocolKotlin
+      - protocolShared
+      - e2e
+      - ciFilters
+  - id: swift-ios
+    target: swift
+    platform: ios
+    app_id: com.shopify.checkoutkit.swiftdemo
+    artifact_env: E2E_SWIFT_IOS_APP_PATH
+    ready_marker: checkout-kit-sample-ready
+    changed_files_filters:
+      - swift
+      - protocolSwift
+      - protocolShared
+      - packageSwift
+      - e2e
+      - ciFilters
+os_version_tags:
+  - latest
+suites:
+  - id: native-preload-cache-hit-preflight
+    execute: tests/preflight/native-preload-cache-hit.yaml

e2e/lib/browserstack_client.rb

@@ -2,19 +2,23 @@
 
 require "net/http"
 require "securerandom"
+require "uri"
 require_relative "../../scripts/lib/json_http_client"
 
 # BrowserStack App Automate API client. Owns endpoint paths, HTTP, and response
 # parsing; callers assemble request payloads and orchestrate build lifecycles.
 class BrowserStackClient
   API_HOST = "api-cloud.browserstack.com"
+  ARTIFACT_HOSTS = [API_HOST, "api.browserstack.com"].freeze
   DASHBOARD_BASE = "https://app-automate.browserstack.com/dashboard/v2/builds"
 
   def self.build_url(build_id)
     build_id.to_s.empty? ? DASHBOARD_BASE : "#{DASHBOARD_BASE}/#{build_id}"
   end
 
   def initialize(username:, access_key:, retries: 0)
+    @username = username
+    @access_key = access_key
     @client = JsonHttpClient.new(
       host: API_HOST,
       error_label: "BrowserStack",
@@ -65,4 +69,32 @@ def stop_build(build_id)
   def get_session(build_id, session_id)
     @client.get("/app-automate/maestro/v2/builds/#{build_id}/sessions/#{session_id}")
   end
+
+  def get_artifact_text(url, redirects_remaining: 3)
+    uri = URI.parse(url)
+    unless uri.scheme == "https" && ARTIFACT_HOSTS.include?(uri.host)
+      raise "BrowserStack artifact URL has an unexpected origin"
+    end
+
+    request = Net::HTTP::Get.new(uri)
+    request.basic_auth(@username, @access_key)
+    response = Net::HTTP.start(
+      uri.host,
+      uri.port,
+      use_ssl: true,
+      open_timeout: 10,
+      read_timeout: 120
+    ) { |http| http.request(request) }
+
+    return response.body.to_s if response.is_a?(Net::HTTPSuccess)
+
+    if response.is_a?(Net::HTTPRedirection) && redirects_remaining.positive?
+      location = response["location"]
+      raise "BrowserStack artifact redirect omitted Location" if location.to_s.empty?
+
+      return get_artifact_text(URI.join(uri, location).to_s, redirects_remaining: redirects_remaining - 1)
+    end
+
+    raise "BrowserStack artifact request failed #{response.code}"
+  end
 end

e2e/lib/e2e_github_reporter.rb

@@ -269,6 +269,11 @@ def failure_details(result)
     tests = result.fetch("failed_tests", [])
     if tests.empty?
       lines << "| — | #{status_icon(result)} | #{artifact_links(nil, result)} |"
+      unless blank?(result["error"])
+        error = result["error"].to_s.gsub(/\s+/, " ").gsub("`", "'")
+        lines << ""
+        lines << "> Diagnostic: `#{error}`"
+      end
     else
       tests.each do |testcase|
         lines << "| `#{testcase.fetch("name", "unknown")}` | ❌ | #{artifact_links(testcase, result)} |"

e2e/scripts/execute_browserstack_run

@@ -18,6 +18,16 @@ class BrowserStackRunExecutor
   # and the version BrowserStack runs cannot drift apart.
   # https://www.browserstack.com/docs/app-automate/maestro/set-up-test-env/configure-tests/set-maestro-version
   MAESTRO_VERSION_FILE = File.expand_path("../.maestro-version", __dir__)
+  PRELOAD_LOG_PREFLIGHT_FLOW = "tests/preflight/native-preload-cache-hit.yaml"
+  PRELOAD_CACHE_HIT_LOGS = {
+    "kotlin" => "Returning cached preloaded WebView."
+  }.freeze
+  ANDROID_CONCURRENT_PRESENTATION_LOG = "Preloaded WebView is already presented; creating a new WebView."
+  BROWSERSTACK_LOG_ARTIFACT_KEYS = %w[device_log instrumentation_log maestro_log].freeze
+  IOS_PRELOAD_DIAGNOSTIC_SIGNALS = [
+    "Presenting cached entry",
+    "Presenting preloaded checkout from cache"
+  ].freeze
 
   def self.failure_result(context, error)
     context.merge(
@@ -82,6 +92,93 @@ class BrowserStackRunExecutor
     value
   end
 
+  def self.build_request_body(run:, app_url:, test_suite_url:, device:, env: ENV)
+    {
+      app: app_url,
+      testSuite: test_suite_url,
+      project: env.fetch("E2E_BROWSERSTACK_PROJECT", "checkout-kit-e2e"),
+      maestroVersion: resolve_maestro_version(env),
+      buildTag: env.fetch("BITRISE_GIT_COMMIT", "local"),
+      customBuildName: run.fetch("id"),
+      devices: [device],
+      execute: [run.fetch("execute")],
+      deviceLogs: true,
+      setEnvVariables: {
+        E2E_APP_ID: run.fetch("app_id"),
+        E2E_READY_MARKER: run.fetch("ready_marker")
+      }
+    }
+  end
+
+  def self.testcases_for(sessions)
+    sessions.flat_map do |session|
+      session.dig("testcases", "data").to_a.flat_map do |group|
+        group.fetch("testcases", [])
+      end
+    end
+  end
+
+  def self.sessions_have_testcases?(sessions)
+    sessions.any? && sessions.all? { |session| testcases_for([session]).any? }
+  end
+
+  # Temporary preflight-only diagnostics: failing iOS Maestro testcases are the
+  # only ones that publish log artifacts, so mine them for the failed steps and
+  # for whether the SDK's cache-hit messages reached the physical device log.
+  # Lines containing URLs are excluded so no checkout values can leak into the
+  # PR report.
+  def self.diagnose_ios_preflight_failure!(run:, sessions:, fetch_log:)
+    return unless run.fetch("execute") == PRELOAD_LOG_PREFLIGHT_FLOW
+    return unless run.fetch("target") == "swift"
+
+    failed = testcases_for(sessions).reject { |testcase| testcase.fetch("status", "") == "passed" }
+    return if failed.empty?
+
+    reports = failed.map do |testcase|
+      maestro_log = testcase["maestro_log"] ? fetch_log.call(testcase["maestro_log"]) : ""
+      device_log = testcase["device_log"] ? fetch_log.call(testcase["device_log"]) : ""
+      failed_steps = maestro_log.lines
+        .select { |line| line.include?("FAILED") && !line.include?("://") }
+        .last(3)
+        .map(&:strip)
+        .join(" | ")
+      signals = IOS_PRELOAD_DIAGNOSTIC_SIGNALS
+        .map { |signal| "#{signal}=#{device_log.include?(signal)}" }
+        .join(", ")
+      "failed steps: #{failed_steps}; device log signals: #{signals}"
+    end
+
+    raise "iOS preload preflight diagnostics — #{reports.join(" || ")}"
+  end
+
+  def self.verify_preload_cache_hit_log!(run:, sessions:, fetch_log:)
+    return unless run.fetch("execute") == PRELOAD_LOG_PREFLIGHT_FLOW
+
+    expected = PRELOAD_CACHE_HIT_LOGS[run.fetch("target")]
+    return unless expected
+
+    passing_testcases = testcases_for(sessions).select do |testcase|
+      testcase.fetch("status", "") == "passed"
+    end
+    log_urls = passing_testcases.flat_map do |testcase|
+      BROWSERSTACK_LOG_ARTIFACT_KEYS.filter_map { |key| testcase[key] }
+    end
+    if log_urls.empty?
+      available_keys = passing_testcases.flat_map(&:keys).uniq.sort.join(", ")
+      raise "Passing BrowserStack testcase did not expose a supported log artifact; available keys: #{available_keys}"
+    end
+
+    device_output = log_urls.map { |url| fetch_log.call(url) }.join("\n")
+    unless device_output.include?(expected)
+      raise "BrowserStack log artifacts did not contain the #{run.fetch("target")} preload cache-hit signal"
+    end
+    if run.fetch("target") == "kotlin" && device_output.include?(ANDROID_CONCURRENT_PRESENTATION_LOG)
+      raise "Android BrowserStack logs recorded a concurrent fresh presentation"
+    end
+
+    puts "Verified #{run.fetch("target")} preload cache-hit signal in BrowserStack logs."
+  end
+
   def initialize(options)
     @options = options
     @client = BrowserStackClient.new(
@@ -102,6 +199,16 @@ class BrowserStackRunExecutor
     puts "BrowserStack build: #{BrowserStackClient.build_url(@build.fetch("build_id"))}"
     build_status = poll_build(@build.fetch("build_id"))
     sessions = fetch_sessions(build_status)
+    self.class.verify_preload_cache_hit_log!(
+      run: @run,
+      sessions: sessions,
+      fetch_log: ->(url) { @client.get_artifact_text(url) }
+    )
+    self.class.diagnose_ios_preflight_failure!(
+      run: @run,
+      sessions: sessions,
+      fetch_log: ->(url) { @client.get_artifact_text(url) }
+    )
     result = normalize_result(@run, @device, @app, @suite, @build, build_status, sessions)
     write_json("result.json", result)
   rescue StandardError => error
@@ -138,20 +245,12 @@ class BrowserStackRunExecutor
   end
 
   def start_build(run, app_url, test_suite_url, device)
-    body = {
-      app: app_url,
-      testSuite: test_suite_url,
-      project: ENV.fetch("E2E_BROWSERSTACK_PROJECT", "checkout-kit-e2e"),
-      maestroVersion: self.class.resolve_maestro_version(ENV),
-      buildTag: ENV.fetch("BITRISE_GIT_COMMIT", "local"),
-      customBuildName: run.fetch("id"),
-      devices: [device],
-      execute: [run.fetch("execute")],
-      setEnvVariables: {
-        E2E_APP_ID: run.fetch("app_id"),
-        E2E_READY_MARKER: run.fetch("ready_marker")
-      }
-    }
+    body = self.class.build_request_body(
+      run: run,
+      app_url: app_url,
+      test_suite_url: test_suite_url,
+      device: device
+    )
     response = @client.start_build(run.fetch("platform"), body)
     write_json("build-start.json", response)
     response
@@ -183,19 +282,31 @@ class BrowserStackRunExecutor
 
   def fetch_sessions(build_status)
     build_id = build_status.fetch("id")
-    build_status.fetch("devices", []).flat_map do |device|
-      device.fetch("sessions", []).map do |session|
-        @client.get_session(build_id, session.fetch("id"))
+    session_ids = build_status.fetch("devices", []).flat_map do |device|
+      device.fetch("sessions", []).map { |session| session.fetch("id") }
+    end
+    deadline = Time.now + ENV.fetch("E2E_BROWSERSTACK_SESSION_DETAILS_TIMEOUT_SECONDS", "120").to_i
+    needs_testcase_artifacts =
+      @run.fetch("execute") == PRELOAD_LOG_PREFLIGHT_FLOW &&
+      PRELOAD_CACHE_HIT_LOGS.key?(@run.fetch("target"))
+
+    loop do
+      sessions = session_ids.map { |session_id| @client.get_session(build_id, session_id) }
+      return sessions unless needs_testcase_artifacts
+      return sessions if self.class.sessions_have_testcases?(sessions)
+
+      if Time.now >= deadline
+        raise "BrowserStack session details did not publish testcase artifacts before timeout"
       end
+
+      sleep ENV.fetch("E2E_BROWSERSTACK_SESSION_DETAILS_POLL_SECONDS", "5").to_i
     end
   end
 
   def normalize_result(run, device, app, suite, build, build_status, sessions)
     status = build_status.fetch("status").to_s.downcase
-    failed_tests = sessions.flat_map do |session|
-      session.dig("testcases", "data").to_a.flat_map do |group|
-        group.fetch("testcases", []).select { |testcase| testcase.fetch("status", "") != "passed" }
-      end
+    failed_tests = self.class.testcases_for(sessions).reject do |testcase|
+      testcase.fetch("status", "") == "passed"
     end
 
     {