From 522faeca4d956688a68f9287170d470d255ca289 Mon Sep 17 00:00:00 2001 From: Kieran Osgood Date: Thu, 30 Jul 2026 13:14:21 +0100 Subject: [PATCH] feat(e2e): select tests by Maestro tag instead of scenario flags The two scenario flags only picked a file, and each test already carries its scenario in its own env block. Tags scale to the tests Phase 6 adds without a new flag per test. Every test now declares tags from one taxonomy: one journey (launch, cart, checkout, account), one cost tier (smoke, full), plus optional quarantine (flaky, wip) and platform capability (ios-only, android-only) tags. maestro_test_tags_test.rb enforces all of it, and a platform tag must name the capability that earns it, so nobody tags an unported test. config.yaml excludes flaky and wip, so a quarantined test stays in the tree and out of every run. `dev rn e2e {ios|android}` now runs the whole tests folder and takes --tags/--exclude-tags. Both match any listed tag, because that is how Maestro filters. The runner also supplies E2E_APP_ID and E2E_READY_MARKER so the shared launch smoke runs locally; a later commit unifies those names. ci.yml gates flow edits on the ruby job, so editing a test reaches the validator. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/ci.yml | 2 + dev.yml | 14 +-- e2e/README.md | 39 ++++++--- e2e/config.yaml | 5 ++ e2e/test/maestro_test_tags_test.rb | 79 +++++++++++++++++ e2e/tests/react-native/checkout-guest.yaml | 3 + .../checkout-hardcoded-buyer-identity.yaml | 3 + e2e/tests/shared/launch-smoke.yaml | 4 + .../react-native/scripts/e2e_maestro_android | 86 ++++++++++--------- .../react-native/scripts/e2e_maestro_ios | 86 ++++++++++--------- 10 files changed, 225 insertions(+), 96 deletions(-) create mode 100644 e2e/test/maestro_test_tags_test.rb diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5b360a66f..5b7aa3f5c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -113,6 +113,8 @@ jobs: - 'e2e/lib/**' - 'e2e/test/**' - 'e2e/scripts/**' + - 'e2e/tests/**' + - 'e2e/config.yaml' - 'e2e/config/matrix.yml' - '.ci/changed-file-filters.yml' - '.github/workflows/ci.yml' diff --git a/dev.yml b/dev.yml index 41edecd57..bb345c17c 100644 --- a/dev.yml +++ b/dev.yml @@ -558,19 +558,19 @@ commands: cd sample/android ./gradlew :shopify_checkout-kit-react-native:test --refresh-dependencies e2e: - desc: Run React Native sample Maestro checkout smoke flows - syntax: "{ios|android} [--guest] [--hardcoded-buyer-identity]" + desc: Run React Native sample Maestro tests, filtered by tag + syntax: "{ios|android} [--tags TAG[,TAG...]] [--exclude-tags TAG[,TAG...]]" run: | - echo "Usage: dev rn e2e {ios|android} [--guest] [--hardcoded-buyer-identity]" >&2 + echo "Usage: dev rn e2e {ios|android} [--tags TAG[,TAG...]] [--exclude-tags TAG[,TAG...]]" >&2 exit 1 subcommands: ios: - desc: Run the React Native iOS Maestro checkout smoke flows - syntax: "[--guest] [--hardcoded-buyer-identity]" + desc: Run the React Native iOS Maestro tests + syntax: "[--tags TAG[,TAG...]] [--exclude-tags TAG[,TAG...]]" run: cd platforms/react-native && ./scripts/e2e_maestro_ios "$@" android: - desc: Run the React Native Android Maestro checkout smoke flows - syntax: "[--guest] [--hardcoded-buyer-identity]" + desc: Run the React Native Android Maestro tests + syntax: "[--tags TAG[,TAG...]] [--exclude-tags TAG[,TAG...]]" run: cd platforms/react-native && ./scripts/e2e_maestro_android "$@" lint: desc: Run all React Native lint checks (Swift, module, sample) diff --git a/e2e/README.md b/e2e/README.md index 254db92f1..64d9da132 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -3,9 +3,9 @@ This directory contains Maestro end-to-end flows and configuration for Checkout Kit sample apps. Two complementary setups live here: -- A **local** React Native checkout smoke suite, run with `dev rn e2e`, that - exercises guest and hardcoded buyer identity checkouts from seeded carts - through Shopify checkout and back to the app. +- A **local** React Native suite, run with `dev rn e2e`, that exercises guest and + hardcoded buyer identity checkouts from seeded carts through Shopify checkout + and back to the app. Tags select which tests run. - A **CI matrix** that expands applications, OS version tags, and suites into BrowserStack Maestro run rows, starting with a shared launch smoke. @@ -30,17 +30,33 @@ React Native Android: dev rn e2e android ``` -Run one or more focused React Native scenarios by passing scenario flags: +Both commands run every test in `tests/`. Narrow a run with `--tags`: ```bash -dev rn e2e ios --guest -dev rn e2e ios --hardcoded-buyer-identity -dev rn e2e ios --guest --hardcoded-buyer-identity -dev rn e2e android --guest -dev rn e2e android --hardcoded-buyer-identity -dev rn e2e android --guest --hardcoded-buyer-identity +dev rn e2e ios --tags checkout +dev rn e2e ios --tags smoke +dev rn e2e android --tags cart,checkout ``` +Both options match **any** listed tag, because that is how Maestro filters. +`--tags cart,checkout` runs the cart tests and the checkout tests. `--exclude-tags` +skips tests carrying any listed tag and defaults to `flaky,wip`. + +### Tags + +Every test declares tags from this taxonomy. `e2e/test/maestro_test_tags_test.rb` +enforces it. + +| Group | Tags | Rule | +|---|---|---| +| Journey | `launch`, `cart`, `checkout`, `account` | Exactly one per test | +| Cost tier | `smoke`, `full` | Exactly one per test | +| Quarantine | `flaky`, `wip` | Excluded by default, in `config.yaml` | +| Platform capability | `ios-only`, `android-only` | Needs a `# Platform capability:` comment | + +A platform tag marks a capability only one platform has, such as Apple Pay. It +must never mark a test that is merely not ported yet. + The React Native commands start Metro if needed, build and launch the target sample app, then run Maestro. They require the standard storefront `.env` setup, but the E2E flows seed their own carts through the bootstrap deep link. The @@ -119,7 +135,8 @@ ruby e2e/scripts/e2e_matrix_to_browserstack_run_plan count ## Files -- `config.yaml` configures Maestro for shared platform behavior. +- `config.yaml` configures Maestro for shared platform behavior and quarantines + the `flaky` and `wip` tags. - `flows/` contains reusable Maestro subflows for app setup and checkout steps. - `tests/react-native/checkout-guest.yaml` composes the React Native guest checkout smoke test from those subflows. diff --git a/e2e/config.yaml b/e2e/config.yaml index 478e48312..113d3a157 100644 --- a/e2e/config.yaml +++ b/e2e/config.yaml @@ -1,6 +1,11 @@ flows: - tests/**/*.yaml +# Quarantined tests stay in the tree and out of every run until someone fixes them. +excludeTags: + - flaky + - wip + platform: ios: # Lets Maestro inspect elements presented inside iOS checkout modal views. diff --git a/e2e/test/maestro_test_tags_test.rb b/e2e/test/maestro_test_tags_test.rb new file mode 100644 index 000000000..c5eb89aca --- /dev/null +++ b/e2e/test/maestro_test_tags_test.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +require "minitest/autorun" +require "yaml" + +class MaestroTestTagsTest < Minitest::Test + E2E_ROOT = File.expand_path("..", __dir__) + JOURNEY_TAGS = ["launch", "cart", "checkout", "account"].freeze + COST_TIER_TAGS = ["smoke", "full"].freeze + QUARANTINE_TAGS = ["flaky", "wip"].freeze + PLATFORM_TAGS = ["ios-only", "android-only"].freeze + KNOWN_TAGS = (JOURNEY_TAGS + COST_TIER_TAGS + QUARANTINE_TAGS + PLATFORM_TAGS).freeze + + def test_files + Dir.glob("tests/**/*.yaml", base: E2E_ROOT).sort + end + + def header(path) + YAML.safe_load(File.read(File.join(E2E_ROOT, path)).split("\n---\n").first) + end + + def tags(path) + header(path)["tags"] || [] + end + + def test_every_test_declares_tags + test_files.each do |path| + refute_empty(tags(path), "#{path} declares no tags, so no CI run can select it") + end + end + + def test_every_tag_belongs_to_the_taxonomy + test_files.each do |path| + tags(path).each do |tag| + assert_includes(KNOWN_TAGS, tag, "#{path} uses the unknown tag #{tag}") + end + end + end + + def test_every_test_declares_one_journey + test_files.each do |path| + journeys = tags(path) & JOURNEY_TAGS + + assert_equal(1, journeys.length, "#{path} must declare exactly one journey tag, found #{journeys.inspect}") + end + end + + def test_every_test_declares_one_cost_tier + test_files.each do |path| + tiers = tags(path) & COST_TIER_TAGS + + assert_equal(1, tiers.length, "#{path} must declare exactly one cost tier tag, found #{tiers.inspect}") + end + end + + def test_a_platform_tag_names_the_capability_that_earns_it + test_files.each do |path| + next if (tags(path) & PLATFORM_TAGS).empty? + + body = File.read(File.join(E2E_ROOT, path)) + + assert_match( + /#\s*Platform capability:\s*\S+/, + body, + "#{path} carries a platform tag, so it must comment `# Platform capability: `" + ) + end + end + + def test_the_workspace_config_quarantines_the_quarantine_tags + config = YAML.safe_load(File.read(File.join(E2E_ROOT, "config.yaml"))) + + assert_equal(QUARANTINE_TAGS.sort, (config["excludeTags"] || []).sort) + end + + def test_there_is_at_least_one_test_to_check + refute_empty(test_files) + end +end diff --git a/e2e/tests/react-native/checkout-guest.yaml b/e2e/tests/react-native/checkout-guest.yaml index 42b72b4ce..639540bee 100644 --- a/e2e/tests/react-native/checkout-guest.yaml +++ b/e2e/tests/react-native/checkout-guest.yaml @@ -1,5 +1,8 @@ appId: ${APP_ID} name: React Native checkout - guest +tags: + - checkout + - full env: # Sample app buyer identity configuration diff --git a/e2e/tests/react-native/checkout-hardcoded-buyer-identity.yaml b/e2e/tests/react-native/checkout-hardcoded-buyer-identity.yaml index 0e82f742a..c126dc48d 100644 --- a/e2e/tests/react-native/checkout-hardcoded-buyer-identity.yaml +++ b/e2e/tests/react-native/checkout-hardcoded-buyer-identity.yaml @@ -1,5 +1,8 @@ appId: ${APP_ID} name: React Native checkout - hardcoded buyer identity +tags: + - checkout + - smoke env: # Sample app buyer identity configuration diff --git a/e2e/tests/shared/launch-smoke.yaml b/e2e/tests/shared/launch-smoke.yaml index 5f534a86b..525235237 100644 --- a/e2e/tests/shared/launch-smoke.yaml +++ b/e2e/tests/shared/launch-smoke.yaml @@ -1,4 +1,8 @@ appId: ${E2E_APP_ID} +name: Launch smoke +tags: + - launch + - smoke --- - launchApp - extendedWaitUntil: diff --git a/platforms/react-native/scripts/e2e_maestro_android b/platforms/react-native/scripts/e2e_maestro_android index 4b06704f7..6daa85df4 100755 --- a/platforms/react-native/scripts/e2e_maestro_android +++ b/platforms/react-native/scripts/e2e_maestro_android @@ -10,42 +10,47 @@ METRO_PID="" E2E_ENV_FILE="" APP_ID="com.shopify.checkoutkit.reactnativedemo" CART_BOOTSTRAP_BASE_LINK="${APP_ID}://e2e/cart?productIndex=0&quantity=1" -MAESTRO_FLOWS=() +READY_MARKER="checkout-kit-sample-ready" +INCLUDE_TAGS="" +EXCLUDE_TAGS="flaky,wip" usage() { cat <&2 -Usage: dev rn e2e android [--guest] [--hardcoded-buyer-identity] +Usage: dev rn e2e android [--tags TAG[,TAG...]] [--exclude-tags TAG[,TAG...]] -Runs all React Native Android Maestro checkout smoke flows against the released native SDK artifacts. -Pass one or more focused scenario flags to run only those flows. +Runs the React Native Android Maestro tests against the released native SDK artifacts. +Without --tags, every test in e2e/tests runs. Tags come from the taxonomy: journey +(launch, cart, checkout, account), cost tier (smoke, full), quarantine (flaky, wip) +and platform capability (ios-only, android-only). Options: - --guest Run only the guest checkout smoke flow. - --hardcoded-buyer-identity Run only the hardcoded buyer identity smoke flow. + --tags TAG[,TAG...] Run only tests carrying any of these tags. + --exclude-tags TAG[,TAG...] Skip tests carrying any of these tags. Defaults to flaky,wip. EOF } -add_maestro_flow() { - local flow="$1" - - if [ "${#MAESTRO_FLOWS[@]}" -gt 0 ]; then - for existing_flow in "${MAESTRO_FLOWS[@]}"; do - if [ "$existing_flow" = "$flow" ]; then - return +while [ "$#" -gt 0 ]; do + case "$1" in + --tags) + INCLUDE_TAGS="${2:-}" + if [ -z "$INCLUDE_TAGS" ]; then + usage + echo "--tags needs a comma separated tag list" >&2 + exit 1 fi - done - fi - - MAESTRO_FLOWS+=("$flow") -} - -for arg in "$@"; do - case "$arg" in - --guest) - add_maestro_flow "tests/react-native/checkout-guest.yaml" + shift 2 + ;; + --tags=*) + INCLUDE_TAGS="${1#--tags=}" + shift ;; - --hardcoded-buyer-identity) - add_maestro_flow "tests/react-native/checkout-hardcoded-buyer-identity.yaml" + --exclude-tags) + EXCLUDE_TAGS="${2:-}" + shift 2 + ;; + --exclude-tags=*) + EXCLUDE_TAGS="${1#--exclude-tags=}" + shift ;; -h|--help) usage @@ -53,19 +58,12 @@ for arg in "$@"; do ;; *) usage - echo "Unknown option: $arg" >&2 + echo "Unknown option: $1" >&2 exit 1 ;; esac done -if [ "${#MAESTRO_FLOWS[@]}" -eq 0 ]; then - MAESTRO_FLOWS=( - "tests/react-native/checkout-guest.yaml" - "tests/react-native/checkout-hardcoded-buyer-identity.yaml" - ) -fi - metro_running() { curl --silent --fail http://localhost:8081/status | grep -q "packager-status:running" } @@ -126,10 +124,20 @@ ENVFILE="$E2E_ENV_FILE" pnpm sample android --extra-params "--refresh-dependenci ( cd "$REPO_ROOT/e2e" - for flow in "${MAESTRO_FLOWS[@]}"; do - "$MAESTRO" --platform android test --config config.yaml \ - -e "APP_ID=${APP_ID}" \ - -e "CART_BOOTSTRAP_BASE_LINK=${CART_BOOTSTRAP_BASE_LINK}" \ - "$flow" - done + MAESTRO_ARGS=(--platform android test --config config.yaml) + + if [ -n "$INCLUDE_TAGS" ]; then + MAESTRO_ARGS+=(--include-tags "$INCLUDE_TAGS") + fi + + if [ -n "$EXCLUDE_TAGS" ]; then + MAESTRO_ARGS+=(--exclude-tags "$EXCLUDE_TAGS") + fi + + "$MAESTRO" "${MAESTRO_ARGS[@]}" \ + -e "APP_ID=${APP_ID}" \ + -e "E2E_APP_ID=${APP_ID}" \ + -e "E2E_READY_MARKER=${READY_MARKER}" \ + -e "CART_BOOTSTRAP_BASE_LINK=${CART_BOOTSTRAP_BASE_LINK}" \ + tests ) diff --git a/platforms/react-native/scripts/e2e_maestro_ios b/platforms/react-native/scripts/e2e_maestro_ios index 2f39cb1ff..2016c0b44 100755 --- a/platforms/react-native/scripts/e2e_maestro_ios +++ b/platforms/react-native/scripts/e2e_maestro_ios @@ -10,42 +10,47 @@ METRO_PID="" E2E_ENV_FILE="" APP_ID="com.shopify.checkoutkit.reactnativedemo" CART_BOOTSTRAP_BASE_LINK="${APP_ID}://e2e/cart?productIndex=0&quantity=1" -MAESTRO_FLOWS=() +READY_MARKER="checkout-kit-sample-ready" +INCLUDE_TAGS="" +EXCLUDE_TAGS="flaky,wip" usage() { cat <&2 -Usage: dev rn e2e ios [--guest] [--hardcoded-buyer-identity] +Usage: dev rn e2e ios [--tags TAG[,TAG...]] [--exclude-tags TAG[,TAG...]] -Runs all React Native iOS Maestro checkout smoke flows against the released native SDK artifacts. -Pass one or more focused scenario flags to run only those flows. +Runs the React Native iOS Maestro tests against the released native SDK artifacts. +Without --tags, every test in e2e/tests runs. Tags come from the taxonomy: journey +(launch, cart, checkout, account), cost tier (smoke, full), quarantine (flaky, wip) +and platform capability (ios-only, android-only). Options: - --guest Run only the guest checkout smoke flow. - --hardcoded-buyer-identity Run only the hardcoded buyer identity smoke flow. + --tags TAG[,TAG...] Run only tests carrying any of these tags. + --exclude-tags TAG[,TAG...] Skip tests carrying any of these tags. Defaults to flaky,wip. EOF } -add_maestro_flow() { - local flow="$1" - - if [ "${#MAESTRO_FLOWS[@]}" -gt 0 ]; then - for existing_flow in "${MAESTRO_FLOWS[@]}"; do - if [ "$existing_flow" = "$flow" ]; then - return +while [ "$#" -gt 0 ]; do + case "$1" in + --tags) + INCLUDE_TAGS="${2:-}" + if [ -z "$INCLUDE_TAGS" ]; then + usage + echo "--tags needs a comma separated tag list" >&2 + exit 1 fi - done - fi - - MAESTRO_FLOWS+=("$flow") -} - -for arg in "$@"; do - case "$arg" in - --guest) - add_maestro_flow "tests/react-native/checkout-guest.yaml" + shift 2 + ;; + --tags=*) + INCLUDE_TAGS="${1#--tags=}" + shift ;; - --hardcoded-buyer-identity) - add_maestro_flow "tests/react-native/checkout-hardcoded-buyer-identity.yaml" + --exclude-tags) + EXCLUDE_TAGS="${2:-}" + shift 2 + ;; + --exclude-tags=*) + EXCLUDE_TAGS="${1#--exclude-tags=}" + shift ;; -h|--help) usage @@ -53,19 +58,12 @@ for arg in "$@"; do ;; *) usage - echo "Unknown option: $arg" >&2 + echo "Unknown option: $1" >&2 exit 1 ;; esac done -if [ "${#MAESTRO_FLOWS[@]}" -eq 0 ]; then - MAESTRO_FLOWS=( - "tests/react-native/checkout-guest.yaml" - "tests/react-native/checkout-hardcoded-buyer-identity.yaml" - ) -fi - metro_running() { curl --silent --fail http://localhost:8081/status | grep -q "packager-status:running" } @@ -127,10 +125,20 @@ ENVFILE="$E2E_ENV_FILE" pnpm sample ios ( cd "$REPO_ROOT/e2e" - for flow in "${MAESTRO_FLOWS[@]}"; do - "$MAESTRO" --platform ios test --config config.yaml \ - -e "APP_ID=${APP_ID}" \ - -e "CART_BOOTSTRAP_BASE_LINK=${CART_BOOTSTRAP_BASE_LINK}" \ - "$flow" - done + MAESTRO_ARGS=(--platform ios test --config config.yaml) + + if [ -n "$INCLUDE_TAGS" ]; then + MAESTRO_ARGS+=(--include-tags "$INCLUDE_TAGS") + fi + + if [ -n "$EXCLUDE_TAGS" ]; then + MAESTRO_ARGS+=(--exclude-tags "$EXCLUDE_TAGS") + fi + + "$MAESTRO" "${MAESTRO_ARGS[@]}" \ + -e "APP_ID=${APP_ID}" \ + -e "E2E_APP_ID=${APP_ID}" \ + -e "E2E_READY_MARKER=${READY_MARKER}" \ + -e "CART_BOOTSTRAP_BASE_LINK=${CART_BOOTSTRAP_BASE_LINK}" \ + tests )