feat(e2e): select tests by Maestro tag instead of scenario flags

.github/workflows/ci.yml

@@ -113,6 +113,8 @@ jobs:
               - 'e2e/lib/**'
               - 'e2e/test/**'
               - 'e2e/scripts/**'
+              - 'e2e/tests/**'
+              - 'e2e/config.yaml'
               - 'e2e/config/matrix.yml'
               - '.ci/changed-file-filters.yml'
               - '.github/workflows/ci.yml'

dev.yml

@@ -558,19 +558,19 @@ commands:
               cd sample/android
               ./gradlew :shopify_checkout-kit-react-native:test --refresh-dependencies
       e2e:
-        desc: Run React Native sample Maestro checkout smoke flows
-        syntax: "{ios|android} [--guest] [--hardcoded-buyer-identity]"
+        desc: Run React Native sample Maestro tests, filtered by tag
+        syntax: "{ios|android} [--tags TAG[,TAG...]] [--exclude-tags TAG[,TAG...]]"
         run: |
-          echo "Usage: dev rn e2e {ios|android} [--guest] [--hardcoded-buyer-identity]" >&2
+          echo "Usage: dev rn e2e {ios|android} [--tags TAG[,TAG...]] [--exclude-tags TAG[,TAG...]]" >&2
           exit 1
         subcommands:
           ios:
-            desc: Run the React Native iOS Maestro checkout smoke flows
-            syntax: "[--guest] [--hardcoded-buyer-identity]"
+            desc: Run the React Native iOS Maestro tests
+            syntax: "[--tags TAG[,TAG...]] [--exclude-tags TAG[,TAG...]]"
             run: cd platforms/react-native && ./scripts/e2e_maestro_ios "$@"
           android:
-            desc: Run the React Native Android Maestro checkout smoke flows
-            syntax: "[--guest] [--hardcoded-buyer-identity]"
+            desc: Run the React Native Android Maestro tests
+            syntax: "[--tags TAG[,TAG...]] [--exclude-tags TAG[,TAG...]]"
             run: cd platforms/react-native && ./scripts/e2e_maestro_android "$@"
       lint:
         desc: Run all React Native lint checks (Swift, module, sample)

e2e/README.md

@@ -3,9 +3,9 @@
 This directory contains Maestro end-to-end flows and configuration for Checkout
 Kit sample apps. Two complementary setups live here:
 
-- A **local** React Native checkout smoke suite, run with `dev rn e2e`, that
-  exercises guest and hardcoded buyer identity checkouts from seeded carts
-  through Shopify checkout and back to the app.
+- A **local** React Native suite, run with `dev rn e2e`, that exercises guest and
+  hardcoded buyer identity checkouts from seeded carts through Shopify checkout
+  and back to the app. Tags select which tests run.
 - A **CI matrix** that expands applications, OS version tags, and suites into
   BrowserStack Maestro run rows, starting with a shared launch smoke.
 
@@ -30,17 +30,33 @@ React Native Android:
 dev rn e2e android
 ```
 
-Run one or more focused React Native scenarios by passing scenario flags:
+Both commands run every test in `tests/`. Narrow a run with `--tags`:
 
 ```bash
-dev rn e2e ios --guest
-dev rn e2e ios --hardcoded-buyer-identity
-dev rn e2e ios --guest --hardcoded-buyer-identity
-dev rn e2e android --guest
-dev rn e2e android --hardcoded-buyer-identity
-dev rn e2e android --guest --hardcoded-buyer-identity
+dev rn e2e ios --tags checkout
+dev rn e2e ios --tags smoke
+dev rn e2e android --tags cart,checkout
 ```
 
+Both options match **any** listed tag, because that is how Maestro filters.
+`--tags cart,checkout` runs the cart tests and the checkout tests. `--exclude-tags`
+skips tests carrying any listed tag and defaults to `flaky,wip`.
+
+### Tags
+
+Every test declares tags from this taxonomy. `e2e/test/maestro_test_tags_test.rb`
+enforces it.
+
+| Group | Tags | Rule |
+|---|---|---|
+| Journey | `launch`, `cart`, `checkout`, `account` | Exactly one per test |
+| Cost tier | `smoke`, `full` | Exactly one per test |
+| Quarantine | `flaky`, `wip` | Excluded by default, in `config.yaml` |
+| Platform capability | `ios-only`, `android-only` | Needs a `# Platform capability:` comment |
+
+A platform tag marks a capability only one platform has, such as Apple Pay. It
+must never mark a test that is merely not ported yet.
+
 The React Native commands start Metro if needed, build and launch the target
 sample app, then run Maestro. They require the standard storefront `.env` setup,
 but the E2E flows seed their own carts through the bootstrap deep link. The
@@ -119,7 +135,8 @@ ruby e2e/scripts/e2e_matrix_to_browserstack_run_plan count
 
 ## Files
 
-- `config.yaml` configures Maestro for shared platform behavior.
+- `config.yaml` configures Maestro for shared platform behavior and quarantines
+  the `flaky` and `wip` tags.
 - `flows/` contains reusable Maestro subflows for app setup and checkout steps.
 - `tests/react-native/checkout-guest.yaml` composes the React Native guest
   checkout smoke test from those subflows.

e2e/config.yaml

@@ -1,6 +1,11 @@
 flows:
   - tests/**/*.yaml
 
+# Quarantined tests stay in the tree and out of every run until someone fixes them.
+excludeTags:
+  - flaky
+  - wip
+
 platform:
   ios:
     # Lets Maestro inspect elements presented inside iOS checkout modal views.

e2e/test/maestro_test_tags_test.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require "minitest/autorun"
+require "yaml"
+
+class MaestroTestTagsTest < Minitest::Test
+  E2E_ROOT = File.expand_path("..", __dir__)
+  JOURNEY_TAGS = ["launch", "cart", "checkout", "account"].freeze
+  COST_TIER_TAGS = ["smoke", "full"].freeze
+  QUARANTINE_TAGS = ["flaky", "wip"].freeze
+  PLATFORM_TAGS = ["ios-only", "android-only"].freeze
+  KNOWN_TAGS = (JOURNEY_TAGS + COST_TIER_TAGS + QUARANTINE_TAGS + PLATFORM_TAGS).freeze
+
+  def test_files
+    Dir.glob("tests/**/*.yaml", base: E2E_ROOT).sort
+  end
+
+  def header(path)
+    YAML.safe_load(File.read(File.join(E2E_ROOT, path)).split("\n---\n").first)
+  end
+
+  def tags(path)
+    header(path)["tags"] || []
+  end
+
+  def test_every_test_declares_tags
+    test_files.each do |path|
+      refute_empty(tags(path), "#{path} declares no tags, so no CI run can select it")
+    end
+  end
+
+  def test_every_tag_belongs_to_the_taxonomy
+    test_files.each do |path|
+      tags(path).each do |tag|
+        assert_includes(KNOWN_TAGS, tag, "#{path} uses the unknown tag #{tag}")
+      end
+    end
+  end
+
+  def test_every_test_declares_one_journey
+    test_files.each do |path|
+      journeys = tags(path) & JOURNEY_TAGS
+
+      assert_equal(1, journeys.length, "#{path} must declare exactly one journey tag, found #{journeys.inspect}")
+    end
+  end
+
+  def test_every_test_declares_one_cost_tier
+    test_files.each do |path|
+      tiers = tags(path) & COST_TIER_TAGS
+
+      assert_equal(1, tiers.length, "#{path} must declare exactly one cost tier tag, found #{tiers.inspect}")
+    end
+  end
+
+  def test_a_platform_tag_names_the_capability_that_earns_it
+    test_files.each do |path|
+      next if (tags(path) & PLATFORM_TAGS).empty?
+
+      body = File.read(File.join(E2E_ROOT, path))
+
+      assert_match(
+        /#\s*Platform capability:\s*\S+/,
+        body,
+        "#{path} carries a platform tag, so it must comment `# Platform capability: <name>`"
+      )
+    end
+  end
+
+  def test_the_workspace_config_quarantines_the_quarantine_tags
+    config = YAML.safe_load(File.read(File.join(E2E_ROOT, "config.yaml")))
+
+    assert_equal(QUARANTINE_TAGS.sort, (config["excludeTags"] || []).sort)
+  end
+
+  def test_there_is_at_least_one_test_to_check
+    refute_empty(test_files)
+  end
+end

e2e/tests/react-native/checkout-guest.yaml

@@ -1,5 +1,8 @@
 appId: ${APP_ID}
 name: React Native checkout - guest
+tags:
+  - checkout
+  - full
 
 env:
   # Sample app buyer identity configuration

e2e/tests/react-native/checkout-hardcoded-buyer-identity.yaml

@@ -1,5 +1,8 @@
 appId: ${APP_ID}
 name: React Native checkout - hardcoded buyer identity
+tags:
+  - checkout
+  - smoke
 
 env:
   # Sample app buyer identity configuration

e2e/tests/shared/launch-smoke.yaml

@@ -1,4 +1,8 @@
 appId: ${E2E_APP_ID}
+name: Launch smoke
+tags:
+  - launch
+  - smoke
 ---
 - launchApp
 - extendedWaitUntil:

platforms/react-native/scripts/e2e_maestro_android

@@ -10,62 +10,60 @@ METRO_PID=""
 E2E_ENV_FILE=""
 APP_ID="com.shopify.checkoutkit.reactnativedemo"
 CART_BOOTSTRAP_BASE_LINK="${APP_ID}://e2e/cart?productIndex=0&quantity=1"
-MAESTRO_FLOWS=()
+READY_MARKER="checkout-kit-sample-ready"
+INCLUDE_TAGS=""
+EXCLUDE_TAGS="flaky,wip"
 
 usage() {
   cat <<EOF >&2
-Usage: dev rn e2e android [--guest] [--hardcoded-buyer-identity]
+Usage: dev rn e2e android [--tags TAG[,TAG...]] [--exclude-tags TAG[,TAG...]]
 
-Runs all React Native Android Maestro checkout smoke flows against the released native SDK artifacts.
-Pass one or more focused scenario flags to run only those flows.
+Runs the React Native Android Maestro tests against the released native SDK artifacts.
+Without --tags, every test in e2e/tests runs. Tags come from the taxonomy: journey
+(launch, cart, checkout, account), cost tier (smoke, full), quarantine (flaky, wip)
+and platform capability (ios-only, android-only).
 
 Options:
-  --guest                       Run only the guest checkout smoke flow.
-  --hardcoded-buyer-identity    Run only the hardcoded buyer identity smoke flow.
+  --tags TAG[,TAG...]           Run only tests carrying any of these tags.
+  --exclude-tags TAG[,TAG...]   Skip tests carrying any of these tags. Defaults to flaky,wip.
 EOF
 }
 
-add_maestro_flow() {
-  local flow="$1"
-
-  if [ "${#MAESTRO_FLOWS[@]}" -gt 0 ]; then
-    for existing_flow in "${MAESTRO_FLOWS[@]}"; do
-      if [ "$existing_flow" = "$flow" ]; then
-        return
+while [ "$#" -gt 0 ]; do
+  case "$1" in
+    --tags)
+      INCLUDE_TAGS="${2:-}"
+      if [ -z "$INCLUDE_TAGS" ]; then
+        usage
+        echo "--tags needs a comma separated tag list" >&2
+        exit 1
       fi
-    done
-  fi
-
-  MAESTRO_FLOWS+=("$flow")
-}
-
-for arg in "$@"; do
-  case "$arg" in
-    --guest)
-      add_maestro_flow "tests/react-native/checkout-guest.yaml"
+      shift 2
+      ;;
+    --tags=*)
+      INCLUDE_TAGS="${1#--tags=}"
+      shift
       ;;
-    --hardcoded-buyer-identity)
-      add_maestro_flow "tests/react-native/checkout-hardcoded-buyer-identity.yaml"
+    --exclude-tags)
+      EXCLUDE_TAGS="${2:-}"
+      shift 2
+      ;;
+    --exclude-tags=*)
+      EXCLUDE_TAGS="${1#--exclude-tags=}"
+      shift
       ;;
     -h|--help)
       usage
       exit 0
       ;;
     *)
       usage
-      echo "Unknown option: $arg" >&2
+      echo "Unknown option: $1" >&2
       exit 1
       ;;
   esac
 done
 
-if [ "${#MAESTRO_FLOWS[@]}" -eq 0 ]; then
-  MAESTRO_FLOWS=(
-    "tests/react-native/checkout-guest.yaml"
-    "tests/react-native/checkout-hardcoded-buyer-identity.yaml"
-  )
-fi
-
 metro_running() {
   curl --silent --fail http://localhost:8081/status | grep -q "packager-status:running"
 }
@@ -126,10 +124,20 @@ ENVFILE="$E2E_ENV_FILE" pnpm sample android --extra-params "--refresh-dependenci
 
 (
   cd "$REPO_ROOT/e2e"
-  for flow in "${MAESTRO_FLOWS[@]}"; do
-    "$MAESTRO" --platform android test --config config.yaml \
-      -e "APP_ID=${APP_ID}" \
-      -e "CART_BOOTSTRAP_BASE_LINK=${CART_BOOTSTRAP_BASE_LINK}" \
-      "$flow"
-  done
+  MAESTRO_ARGS=(--platform android test --config config.yaml)
+
+  if [ -n "$INCLUDE_TAGS" ]; then
+    MAESTRO_ARGS+=(--include-tags "$INCLUDE_TAGS")
+  fi
+
+  if [ -n "$EXCLUDE_TAGS" ]; then
+    MAESTRO_ARGS+=(--exclude-tags "$EXCLUDE_TAGS")
+  fi
+
+  "$MAESTRO" "${MAESTRO_ARGS[@]}" \
+    -e "APP_ID=${APP_ID}" \
+    -e "E2E_APP_ID=${APP_ID}" \
+    -e "E2E_READY_MARKER=${READY_MARKER}" \
+    -e "CART_BOOTSTRAP_BASE_LINK=${CART_BOOTSTRAP_BASE_LINK}" \
+    tests
 )