diff --git a/dev.yml b/dev.yml index 25cd9d867..f81cc8430 100644 --- a/dev.yml +++ b/dev.yml @@ -70,6 +70,7 @@ check: generate-env-tests: ./scripts/test_generate_env_files storefront-env-tests: ./scripts/test_setup_storefront_env run-maestro-tests: ./e2e/scripts/test_run_maestro + bitrise-ci-helper-tests: ./e2e/scripts/test_bitrise_ci_helpers ruby-script-tests: ./scripts/test_ruby android-detekt: platforms/android/gradlew -p platforms/android detekt android-lint: | diff --git a/e2e/FLAKES.md b/e2e/FLAKES.md index 025e0dce4..340acf826 100644 --- a/e2e/FLAKES.md +++ b/e2e/FLAKES.md @@ -22,7 +22,7 @@ They block the whole matrix. Remove this section when the matrix reports green. | # | Target | Symptom | Suspected cause | |---|---|---|---| | B1 | Swift iOS and React Native iOS | Every test that opens the control link fails 1 second after launch. `launch-smoke` passes, so the app starts and the ready marker appears. | Fixed, and proved by two probe builds. BrowserStack ran Maestro 1.39.10, which drives iOS `openLink` through `xcrun simctl openurl`. That command accepts a simulator only, so a real device answers `Invalid device` and exit code 148. BrowserStack added iOS `openLink` in 2.0.7, so the build sends `maestroVersion` from `e2e/.maestro-version`. Maestro 2.4.0 alone still fails, because it implements iOS `clearState` by uninstalling the app and the BrowserStack reinstall reports success without restoring the app. Eleven measured builds separate the two faults: eight with `clearState` failed on five different units, and three without it passed, two of them on units that had just failed. The flag is therefore deterministic, not flaky, and a retry cannot recover it. With both changes `cart-from-control-link` passes in 17 seconds and `openLink` returns in 36 milliseconds through the XCTest driver. | -| B2 | Kotlin Android and React Native Android | `checkout-hardcoded-buyer-identity` fails 2 to 6 minutes in. `launch-smoke`, `cart-from-control-link` and `checkout-present-and-close` all pass, so the control link and the cart work. | Fixed. Two separate faults. First, the flow matched `^First name$` and the field reads `First name (optional)`, so the tap timed out. Second, Bitrise holds no address secrets, so `scripts/setup_storefront_env` fell back to `Toronto, ON, M5V 1M7, CA`. A Canadian address sets the billing country, the province list, the postal field label and the currency, and the flow expects `ZIP code`. The defaults are now the US fixture. Both targets now pass. | +| B2 | Kotlin Android and React Native Android | `checkout-hardcoded-buyer-identity` fails 2 to 6 minutes in. `launch-smoke`, `cart-from-control-link` and `checkout-present-and-close` all pass, so the control link and the cart work. | Fixed. Two separate faults. First, the flow matched `^First name$` and the field reads `First name (optional)`, so the tap timed out. Second, Bitrise holds no address secrets, so `scripts/setup_storefront_env` fell back to `Toronto, ON, M5V 1M7, CA`. A Canadian address sets the billing country, the province list, the postal field label and the currency, and the flow expects `ZIP code`. The defaults are now the US fixture. Both targets now pass. That fix relied on CI holding no address values at all, which encrypted config later changed. Two guards now hold it instead: the suite builds from `e2e/.env`, which carries no address, and it passes `--ignore-generated` so a stale demo address in generated platform config cannot win. `scripts/test_setup_storefront_env` and `e2e/scripts/test_bitrise_ci_helpers` cover both. | | B3 | Kotlin Android and React Native Android | `checkout-customer-account` fails after 69 seconds, at `Assert that "^Email( address)?$" is visible`. | Confirmed, and it is a missing secret rather than a defect. The sign-in web view opens `http://null/oauth/authorize` with an empty `client_id`, because Bitrise supplies no `CUSTOMER_ACCOUNT_API_CLIENT_ID` and no `CUSTOMER_ACCOUNT_API_SHOP_ID`. The sample reads both at build time. `account_enabled?` now requires all four account values, so the run excludes the account tag until someone adds the two secrets. | | B4 | Kotlin Android | `checkout-guest` fails after 127 seconds, at `Scrolling DOWN until "Country/Region" is visible`. React Native Android passes the same test in 358 seconds. | Confirmed, and it is a sample defect rather than a BrowserStack fault, so it reproduces anywhere. The Android sample gives a guest cart a Canadian buyer identity: `CartRepository.kt:102` returns `CartBuyerIdentityInput(countryCode = CA)`. Swift returns `CartInput(lines:)` and React Native returns `{}`, so both take the shop market. Checkout then renders a Canadian form and totals in CAD, while the shared fixture is a United States address. The label node matches the text but reports `height=0` and the same bounds through all six swipes, so the scroll can never satisfy it. The React Native log proves the contrast: `United States` is already visible there, so the country picker step is skipped. | @@ -37,6 +37,14 @@ They block the whole matrix. Remove this section when the matrix reports green. These produce a wrong verdict rather than an unstable one. The flows already avoid them. Keep them out of new flows. +- The buyer address decides the billing country, the province list, the postal field + label and the currency. The checkout flows assert `ZIP code`, `State` and + `California`, so any address outside the United States fails them. Change the + address and the flows together, never one alone. +- `scripts/setup_storefront_env` reads the files it generates as a value source, so a + run can inherit whatever the previous run wrote. The suite passes + `--ignore-generated` for that reason. A new caller that omits it can pick up the + demo store on a workspace that already built the demo app. - `visibilityPercentage` under 100 rounds down to zero. `scrollUntilVisible` then stops before it moves, and the tap that follows lands on whatever already sat there. Use `centerElement: true`. diff --git a/e2e/README.md b/e2e/README.md index 8bec7178f..77716ccba 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -16,6 +16,17 @@ Both paths call `scripts/run_maestro`, so they share one environment contract. The suite reads its own file, `e2e/.env`, so a run cannot pick up whichever store you happen to have configured for the sample apps in the repo-root `.env`. +CI builds the sample apps from that same file. `e2e_configure_storefront` calls +`scripts/setup_storefront_env --env-file e2e/.env --ignore-generated`, so the app under +test talks to the E2E store and never to the demo store. `--ignore-generated` stops the +previously generated platform config acting as a value source, because on a workspace +that already built the demo app it holds the demo store and a Canadian address. + +`e2e.ejson` holds no buyer address. The address comes from the defaults in +`scripts/setup_storefront_env`, which are in the United States, because the checkout +flows assert on `ZIP code`, `State` and `California`. Change one and you must change the +other. See flake B2 in [FLAKES.md](FLAKES.md). + `dev up` generates `e2e/.env` from `config/secrets/e2e.ejson`, which is committed encrypted. It is generated, so an edit to it is lost on the next `dev up`. To change a value: @@ -180,6 +191,10 @@ ruby e2e/scripts/e2e_matrix_to_browserstack_run_plan count It holds the environment contract and the workspace root rule in one place. - `scripts/test_run_maestro` puts a fake `maestro` on `PATH` and asserts the argv, so the environment contract has tests that need no device. +- `scripts/bitrise_ci_helpers` holds the shared shell functions the CI build steps use, + including `e2e_configure_storefront`. +- `scripts/test_bitrise_ci_helpers` puts a fake `setup_storefront_env` on the path and + asserts the argv, so the split between the E2E store and the demo store has a test. - `config/matrix.yml`, `lib/e2e_matrix_to_browserstack_run_plan.rb`, and `scripts/` drive the BrowserStack run plan. diff --git a/e2e/scripts/bitrise_ci_helpers b/e2e/scripts/bitrise_ci_helpers index 462217f18..d52faed7b 100755 --- a/e2e/scripts/bitrise_ci_helpers +++ b/e2e/scripts/bitrise_ci_helpers @@ -139,11 +139,19 @@ e2e_generate_env_files() { ./scripts/generate_env_files } +# The suite has its own shop, in config/secrets/e2e.ejson. The repo-root .env +# comes from config/secrets/demo.ejson and a developer may point it at their own +# shop, so reading it here would run the suite against whichever store happened +# to be configured for manual testing. +# +# e2e/.env carries no buyer address, so scripts/setup_storefront_env falls back to +# its own defaults. Those defaults are in the United States, which is what the +# checkout flows assert on. See flake B2 in e2e/FLAKES.md. e2e_configure_storefront() { e2e_generate_env_files - e2e_log "Configuring storefront environment" - ./scripts/setup_storefront_env + e2e_log "Configuring storefront environment from e2e/.env" + ./scripts/setup_storefront_env --env-file e2e/.env --ignore-generated } # execute_browserstack_run reads these from the process environment, and the diff --git a/e2e/scripts/test_bitrise_ci_helpers b/e2e/scripts/test_bitrise_ci_helpers new file mode 100755 index 000000000..0ac8d01bc --- /dev/null +++ b/e2e/scripts/test_bitrise_ci_helpers @@ -0,0 +1,138 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Synthetic tests for e2e/scripts/bitrise_ci_helpers. +# +# The guarantee here is the separation between the two stores. The sample apps +# are configured from the repo-root .env, which comes from config/secrets/demo.ejson +# and which a developer may point at their own shop. The E2E suite has its own +# shop in config/secrets/e2e.ejson. A suite run that read the repo-root .env would +# test whichever store happened to be configured for manual work. +# +# scripts/test_setup_storefront_env proves --env-file reads the file it is given. +# This file proves the E2E path passes it. +# +# The helpers only define functions when sourced, so no CI environment is needed. + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +HELPERS="${REPO_ROOT}/e2e/scripts/bitrise_ci_helpers" + +fixtures=() + +cleanup() { + local fixture + if [[ "${#fixtures[@]}" -eq 0 ]]; then + return + fi + + for fixture in "${fixtures[@]}"; do + rm -rf "$fixture" + done +} + +trap cleanup EXIT + +fail() { + echo "test_bitrise_ci_helpers: $1" >&2 + exit 1 +} + +# A fake scripts/setup_storefront_env that records its argv instead of writing +# any platform configuration. +make_fixture() { + local fixture + fixture="$(mktemp -d "${TMPDIR:-/tmp}/checkout-kit-ci-helpers.XXXXXX")" + fixtures+=("$fixture") + + mkdir -p "$fixture/scripts" "$fixture/e2e" + + cat >"$fixture/scripts/setup_storefront_env" <<'EOF' +#!/usr/bin/env bash +printf '%s\n' "$@" >"${ARGV_LOG}" +EOF + chmod +x "$fixture/scripts/setup_storefront_env" + + printf '%s\n' "$fixture" +} + +# Runs e2e_configure_storefront with the decrypt step stubbed out, so the test +# needs no key and no network. +run_configure_storefront() { + local fixture="$1" + + ( + source "$HELPERS" + + e2e_generate_env_files() { :; } + e2e_log() { :; } + + cd "$fixture" + ARGV_LOG="$fixture/argv.log" e2e_configure_storefront + ) +} + +argv_of() { + cat "$1/argv.log" +} + +test_configure_storefront_reads_the_e2e_env_file() { + local fixture argv + fixture="$(make_fixture)" + + run_configure_storefront "$fixture" + + [[ -f "$fixture/argv.log" ]] || fail "setup_storefront_env was never called" + + argv="$(argv_of "$fixture")" + + grep -Fxq -- "--env-file" <<<"$argv" || + fail "e2e_configure_storefront did not pass --env-file, so the suite reads the repo-root .env" + + grep -Fxq -- "e2e/.env" <<<"$argv" || + fail "e2e_configure_storefront passed --env-file but not e2e/.env" +} + +# The flag and its value must be adjacent, or the value lands on some other flag. +test_env_file_value_follows_the_flag() { + local fixture + fixture="$(make_fixture)" + + run_configure_storefront "$fixture" + + grep -A 1 -Fx -- "--env-file" "$fixture/argv.log" | grep -Fxq -- "e2e/.env" || + fail "the value after --env-file is not e2e/.env" +} + +# Generated platform config is a value source, and on a workspace that already +# built the demo app it holds the demo store and a Canadian address. The suite +# must not inherit it. See flake B2 in e2e/FLAKES.md. +test_configure_storefront_ignores_generated_config() { + local fixture + fixture="$(make_fixture)" + + run_configure_storefront "$fixture" + + grep -Fxq -- "--ignore-generated" "$fixture/argv.log" || + fail "e2e_configure_storefront did not pass --ignore-generated, so a stale demo address can win" +} + +test_configure_storefront_never_names_the_repo_root_env() { + local fixture line + fixture="$(make_fixture)" + + run_configure_storefront "$fixture" + + while IFS= read -r line; do + if [[ "$line" == ".env" || "$line" == "./.env" || "$line" == */checkout-kit/.env ]]; then + fail "e2e_configure_storefront pointed setup_storefront_env at the repo-root .env" + fi + done <"$fixture/argv.log" +} + +test_configure_storefront_reads_the_e2e_env_file +test_env_file_value_follows_the_flag +test_configure_storefront_ignores_generated_config +test_configure_storefront_never_names_the_repo_root_env + +echo "bitrise_ci_helpers synthetic tests passed." diff --git a/scripts/setup_storefront_env b/scripts/setup_storefront_env index b4b8b4488..1f7b8eb45 100755 --- a/scripts/setup_storefront_env +++ b/scripts/setup_storefront_env @@ -4,8 +4,6 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -ROOT_ENV="${ROOT_DIR}/.env" -ROOT_ENV_LOCAL="${ROOT_DIR}/.env.local" ANDROID_ENV="${ROOT_DIR}/platforms/android/samples/CheckoutKitAndroidDemo/.env" SWIFT_DEMO_XCCONFIG="${ROOT_DIR}/platforms/swift/Samples/CheckoutKitSwiftDemo/Storefront.xcconfig" SWIFT_ACCELERATED_XCCONFIG="${ROOT_DIR}/platforms/swift/Samples/ShopifyAcceleratedCheckoutsApp/Storefront.xcconfig" @@ -16,11 +14,20 @@ DEFAULT_CUSTOMER_ACCOUNT_API_VERSION="2026-04" usage() { cat <&2 + exit 1 + fi + env_file="$2" + shift + ;; + --ignore-generated) + IGNORE_GENERATED="true" + ;; -h | --help) usage exit 0 @@ -49,6 +69,14 @@ while [[ $# -gt 0 ]]; do shift done +ROOT_ENV="$env_file" +ROOT_ENV_LOCAL="${env_file}.local" + +# Messages name the files the caller actually passed, so an E2E run never tells a +# developer to edit .env.local when it is reading e2e/.env.local. +ENV_LABEL="${ROOT_ENV#"${ROOT_DIR}/"}" +ENV_LOCAL_LABEL="${ROOT_ENV_LOCAL#"${ROOT_DIR}/"}" + trim() { sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' } @@ -71,6 +99,21 @@ strip_outer_quotes() { printf '%s' "$value" } +# Generated platform config doubles as a value source, so a developer who lost +# .env keeps their store instead of silently blanking it. That inheritance is +# wrong for a caller that wants configuration to be a pure function of the file +# it passed: the generated files hold whatever the previous run wrote, which for +# an E2E run is the demo store. --ignore-generated drops them as a source. +is_generated_file() { + case "$1" in + "$ANDROID_ENV" | "$REACT_NATIVE_ENV" | "$SWIFT_DEMO_XCCONFIG" | "$SWIFT_ACCELERATED_XCCONFIG") + return 0 + ;; + esac + + return 1 +} + read_env_value() { local key="$1" local file="$2" @@ -80,6 +123,10 @@ read_env_value() { return 0 fi + if [[ "$IGNORE_GENERATED" == "true" ]] && is_generated_file "$file"; then + return 0 + fi + raw_value="$(awk -v key="$key" ' /^[[:space:]]*#/ || /^[[:space:]]*\/\// || /^[[:space:]]*$/ { next } $0 !~ /=/ { next } @@ -172,8 +219,8 @@ report_local_overrides() { keys="$(env_keys "$ROOT_ENV_LOCAL" | paste -sd, - | sed -e 's/,/, /g')" [[ -n "$keys" ]] || return 0 - echo ".env.local overrides .env for: ${keys}" - echo "Nothing writes to .env.local, so remove a line there to fall back to .env." + echo "${ENV_LOCAL_LABEL} overrides ${ENV_LABEL} for: ${keys}" + echo "Nothing writes to ${ENV_LOCAL_LABEL}, so remove a line there to fall back to ${ENV_LABEL}." } is_missing_required_value() { @@ -412,11 +459,11 @@ load_values() { # checkout builds; the sample app then fails at its first network call instead. report_missing_required_values() { if is_missing_required_value "$STOREFRONT_DOMAIN_VALUE"; then - echo "No usable STOREFRONT_DOMAIN. Set it in .env.local, or run \`dev up\` to generate .env." >&2 + echo "No usable STOREFRONT_DOMAIN. Set it in ${ENV_LOCAL_LABEL}, or run \`dev up\` to generate ${ENV_LABEL}." >&2 fi if is_missing_required_value "$STOREFRONT_ACCESS_TOKEN_VALUE"; then - echo "No usable STOREFRONT_ACCESS_TOKEN. Set it in .env.local, or run \`dev up\` to generate .env." >&2 + echo "No usable STOREFRONT_ACCESS_TOKEN. Set it in ${ENV_LOCAL_LABEL}, or run \`dev up\` to generate ${ENV_LABEL}." >&2 fi } @@ -435,16 +482,16 @@ write_xcconfig_assignment() { generate_platform_env_header() { cat <"$fixture/e2e/.env" <<'EOF' +STOREFRONT_DOMAIN=e2e-store.example.myshopify.com +STOREFRONT_ACCESS_TOKEN=e2e-token +API_VERSION=2026-04 +CUSTOMER_ACCOUNT_API_VERSION=2026-04 +EOF + + "$fixture/scripts/setup_storefront_env" --env-file "$fixture/e2e/.env" >"$output" 2>&1 + assert_output_is_sanitized "$output" + + assert_file_is_unchanged "$fixture/.env" "$root_before" + assert_contains "$android_env" "STOREFRONT_DOMAIN=e2e-store.example.myshopify.com" + assert_not_contains "$android_env" "synthetic-store.example.myshopify.com" +} + +# Guards flake B2. The E2E file carries no address, so the defaults decide it. The +# checkout flows assert on "ZIP code", "State" and "California", so a Canadian +# address from the repo-root .env fails them on every run. +test_env_file_address_defaults_stay_in_the_united_states() { + local fixture output android_env + fixture="$(make_fixture)" + output="$fixture/output.log" + android_env="$(android_env_of "$fixture")" + + write_canonical_root_env "$fixture/.env" synthetic-store.example.myshopify.com + + mkdir -p "$fixture/e2e" + cat >"$fixture/e2e/.env" <<'EOF' +STOREFRONT_DOMAIN=e2e-store.example.myshopify.com +STOREFRONT_ACCESS_TOKEN=e2e-token +EOF + + "$fixture/scripts/setup_storefront_env" --env-file "$fixture/e2e/.env" >"$output" 2>&1 + assert_output_is_sanitized "$output" + + assert_contains "$android_env" "CITY=Los Angeles" + assert_contains "$android_env" "COUNTRY=US" + assert_contains "$android_env" "PROVINCE=CA" + assert_contains "$android_env" "ZIP=90017" + assert_not_contains "$android_env" "Toronto" + assert_not_contains "$android_env" "M5V 1M7" +} + +test_env_file_takes_its_own_local_overlay() { + local fixture output android_env + fixture="$(make_fixture)" + output="$fixture/output.log" + android_env="$(android_env_of "$fixture")" + + write_canonical_root_env "$fixture/.env" synthetic-store.example.myshopify.com + printf 'STOREFRONT_DOMAIN=%s\n' overridden-store.example.myshopify.com >"$fixture/.env.local" + + mkdir -p "$fixture/e2e" + cat >"$fixture/e2e/.env" <<'EOF' +STOREFRONT_DOMAIN=e2e-store.example.myshopify.com +STOREFRONT_ACCESS_TOKEN=e2e-token +EOF + printf 'STOREFRONT_DOMAIN=%s\n' e2e-local-store.example.myshopify.com >"$fixture/e2e/.env.local" + + "$fixture/scripts/setup_storefront_env" --env-file "$fixture/e2e/.env" >"$output" 2>&1 + assert_output_is_sanitized "$output" + + assert_contains "$android_env" "STOREFRONT_DOMAIN=e2e-local-store.example.myshopify.com" + assert_not_contains "$android_env" "overridden-store.example.myshopify.com" +} + +# Guards flake B2 against its second route. Generated platform config is a +# fallback so a developer who lost .env keeps their values, but for an E2E run it +# holds stale demo values. A Canadian city there would beat the United States +# default and fail the checkout flows, exactly as it did the first time. +test_env_file_ignores_stale_generated_config_for_the_address() { + local fixture output android_env react_native_env + fixture="$(make_fixture)" + output="$fixture/output.log" + android_env="$(android_env_of "$fixture")" + react_native_env="$(react_native_env_of "$fixture")" + + cat >"$react_native_env" <<'EOF' +CITY=Toronto +COUNTRY=CA +PROVINCE=ON +ZIP=M5V 1M7 +ADDRESS_1=650 King Street +EOF + + mkdir -p "$fixture/e2e" + cat >"$fixture/e2e/.env" <<'EOF' +STOREFRONT_DOMAIN=e2e-store.example.myshopify.com +STOREFRONT_ACCESS_TOKEN=e2e-token +EOF + + "$fixture/scripts/setup_storefront_env" \ + --env-file "$fixture/e2e/.env" --ignore-generated >"$output" 2>&1 + assert_output_is_sanitized "$output" + + assert_contains "$android_env" "CITY=Los Angeles" + assert_contains "$android_env" "COUNTRY=US" + assert_contains "$android_env" "ZIP=90017" + assert_not_contains "$android_env" "Toronto" + assert_not_contains "$android_env" "M5V 1M7" +} + +# The demo path keeps inheriting, so a developer who lost .env does not silently +# lose their configured store. +test_generated_config_is_still_inherited_without_the_flag() { + local fixture output android_env react_native_env + fixture="$(make_fixture)" + output="$fixture/output.log" + android_env="$(android_env_of "$fixture")" + react_native_env="$(react_native_env_of "$fixture")" + + cat >"$react_native_env" <<'EOF' +CITY=Toronto +COUNTRY=CA +EOF + + mkdir -p "$fixture/e2e" + cat >"$fixture/e2e/.env" <<'EOF' +STOREFRONT_DOMAIN=e2e-store.example.myshopify.com +STOREFRONT_ACCESS_TOKEN=e2e-token +EOF + + "$fixture/scripts/setup_storefront_env" --env-file "$fixture/e2e/.env" >"$output" 2>&1 + assert_output_is_sanitized "$output" + + assert_contains "$android_env" "CITY=Toronto" +} + +test_env_file_requires_a_path() { + local fixture output + fixture="$(make_fixture)" + output="$fixture/output.log" + + if "$fixture/scripts/setup_storefront_env" --env-file >"$output" 2>&1; then + fail "--env-file was accepted with no path" + fi + assert_contains "$output" "Usage:" +} + # rather than have it silently ignored. test_rejects_the_removed_prompt_flags() { local fixture output flag @@ -604,6 +760,12 @@ test_env_local_overrides_root_env test_env_local_warning_names_only_keys test_env_local_is_never_baked_into_root_env test_merchant_identifier_is_absent_from_generated_config +test_env_file_selects_an_alternate_source +test_env_file_address_defaults_stay_in_the_united_states +test_env_file_takes_its_own_local_overlay +test_env_file_ignores_stale_generated_config_for_the_address +test_generated_config_is_still_inherited_without_the_flag +test_env_file_requires_a_path test_rejects_the_removed_prompt_flags echo "setup_storefront_env synthetic tests passed."