diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a150fe..2a7c261 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,5 @@ -# Continuous integration: static checks first, then API and UI tests in parallel jobs, -# then one merged HTML report for the whole run. +# Continuous integration: static checks first, then the API and UI tests split into parallel +# shards, then one result per project (api, chromium, firefox, webkit) and one merged HTML report. # # Runs on every push and pull request to main, by hand (Actions tab -> Run workflow), # and every night: the shop under test is a public demo that can change without any @@ -36,24 +36,35 @@ jobs: - run: npm ci - run: npm run check - tests: - name: Tests (${{ matrix.project }}) + # Each browser's UI tests are split across 2 machines with Playwright's --shard; the API + # tests take about a minute, so they stay in one job. More shards also mean more load on the + # public demo shop (2 workers each), so raise `total` only when a project gets slow. + # Keep the shard counts here in step with `shards` in the tests job below. + shards: + name: Shard (${{ matrix.project }} ${{ matrix.shard }}/${{ matrix.total }}) needs: checks # no point starting browsers if the code does not even compile runs-on: ubuntu-latest timeout-minutes: 20 strategy: - fail-fast: false # one browser failing should not cancel the others + fail-fast: false # one shard failing should not cancel the others matrix: - project: [api, chromium, firefox, webkit] + include: + - { project: api, shard: 1, total: 1 } + - { project: chromium, shard: 1, total: 2 } + - { project: chromium, shard: 2, total: 2 } + - { project: firefox, shard: 1, total: 2 } + - { project: firefox, shard: 2, total: 2 } + - { project: webkit, shard: 1, total: 2 } + - { project: webkit, shard: 2, total: 2 } env: - # Names this job's blob report (see playwright.config.ts). - BLOB_NAME: ${{ matrix.project }} + # Names this shard's blob report, e.g. report-chromium-1.zip (see playwright.config.ts). + BLOB_NAME: ${{ matrix.project }}-${{ matrix.shard }} steps: - uses: actions/checkout@v7 - uses: actions/setup-node@v7 with: node-version: 24 - cache: npm + cache: npm # reuses downloaded packages between runs (keyed on package-lock.json) - run: npm ci # Only the browser this job needs (with its system libraries); the API job needs none. @@ -61,18 +72,63 @@ jobs: if: matrix.project != 'api' run: npx playwright install --with-deps ${{ matrix.project }} + # --shard=1/2 runs the first half of the test files of this project, 2/2 the rest. - name: Run tests - run: npx playwright test --project=${{ matrix.project }} + run: npx playwright test --project=${{ matrix.project }} --shard=${{ matrix.shard }}/${{ matrix.total }} # Upload even when tests failed: that is exactly when the report is needed. - name: Upload blob report if: ${{ !cancelled() }} uses: actions/upload-artifact@v7 with: - name: blob-report-${{ matrix.project }} + name: blob-report-${{ matrix.project }}-${{ matrix.shard }} path: blob-report retention-days: 3 + # One job per project merges that project's shards into a single result, so the pull + # request checks (and the "Protect main" ruleset) still see Tests (api), Tests (chromium), + # Tests (firefox) and Tests (webkit), and the artifacts keep their unsharded names. + tests: + name: Tests (${{ matrix.project }}) + needs: shards + if: ${{ !cancelled() }} # also runs when a shard failed: this job reports it + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - { project: api, shards: 1 } + - { project: chromium, shards: 2 } + - { project: firefox, shards: 2 } + - { project: webkit, shards: 2 } + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 + with: + node-version: 24 + cache: npm + - run: npm ci + + - name: Download the blob reports of ${{ matrix.project }} + uses: actions/download-artifact@v8 + with: + path: blob-reports + pattern: blob-report-${{ matrix.project }}-* + merge-multiple: true + + # The same JUnit XML and JSON files one unsharded job would write (test names, retries, + # flaky tests unchanged). Skipped when no shard uploaded anything; the check below says so. + - name: Merge the shards + if: ${{ hashFiles('blob-reports/*.zip') != '' }} + env: + PLAYWRIGHT_JUNIT_OUTPUT_FILE: test-results/junit.xml + PLAYWRIGHT_JSON_OUTPUT_FILE: test-results/results.json + run: npx playwright merge-reports --reporter junit,json ./blob-reports + + # Fails this job when a shard is missing or a test failed (see the script's comment). + - name: Check the merged results + run: node src/ci/check-merged-results.mts blob-reports ${{ matrix.shards }} test-results/results.json + # JUnit XML only, as its own artifact: the CI results collection of # automation-coverage-quality-intelligence imports it (config/producers.toml there) - name: Keep the JUnit results @@ -97,7 +153,7 @@ jobs: report: name: Merge HTML report - needs: tests + needs: [shards, tests] if: ${{ !cancelled() }} runs-on: ubuntu-latest steps: @@ -115,7 +171,7 @@ jobs: pattern: blob-report-* merge-multiple: true - # One HTML report with every project (api, chromium, firefox, webkit) of this run. + # One HTML report with every shard of every project (api, chromium, firefox, webkit) of this run. - name: Merge into one HTML report run: npx playwright merge-reports --reporter html ./all-blob-reports diff --git a/README.md b/README.md index a8594ee..d8a6a71 100644 --- a/README.md +++ b/README.md @@ -21,14 +21,14 @@ a demo web shop built for testers (Angular front end + REST API), written with ## Tech stack -| | | -| ------------- | ---------------------------------------------------------------------------------------------------------- | -| Test runner | Playwright Test 1.64 (TypeScript, no extra compiler step) | -| Language | TypeScript 6.0, strict mode | -| API contracts | zod 4 schemas, which also provide the TypeScript types | -| Static checks | `tsc`, ESLint (typescript-eslint, type-aware + eslint-plugin-playwright), Prettier | -| CI | GitHub Actions: checks, then 4 parallel jobs (api, chromium, firefox, webkit), then one merged HTML report | -| Node | 20.12 or newer (CI uses 24) | +| | | +| ------------- | --------------------------------------------------------------------------------------------------------------------------------- | +| Test runner | Playwright Test 1.64 (TypeScript, no extra compiler step) | +| Language | TypeScript 6.0, strict mode | +| API contracts | zod 4 schemas, which also provide the TypeScript types | +| Static checks | `tsc`, ESLint (typescript-eslint, type-aware + eslint-plugin-playwright), Prettier | +| CI | GitHub Actions: checks, then 7 parallel shards (`--shard`; api 1, each browser 2), one result per project, one merged HTML report | +| Node | 20.12 or newer (CI uses 24) | ## Quick start @@ -178,11 +178,21 @@ snapshots, network and console) and a page snapshot (`error-context.md`). On a r (the demo can change without a commit here): 1. **checks**: `npm run check` (typecheck, ESLint, Prettier). Nothing else starts if this fails. -2. **tests**: four parallel jobs, `api`, `chromium`, `firefox`, `webkit`; each installs only its own browser and uploads - a blob report, also when tests fail. -3. **report**: merges the blob reports into one HTML report for all four projects (artifact `playwright-report`). - -On CI there are 2 retries, 2 workers per job, and `forbidOnly`, so a forgotten `test.only` fails the build. Each test +2. **shards**: seven parallel jobs. The API tests run in one job; the UI tests of each browser are split across two + machines with Playwright's `--shard=1/2` and `--shard=2/2`. Each job installs only its own browser and uploads a blob + report, also when tests fail. +3. **tests**: one job per project (`Tests (api)`, `Tests (chromium)`, ...) merges that project's shards with + `playwright merge-reports` into the same JUnit XML and JSON report an unsharded run writes, then + [`src/ci/check-merged-results.mts`](src/ci/check-merged-results.mts) fails the job if a shard is missing or a test + failed. So the pull request checks and the artifact names did not change when sharding came in. +4. **report**: merges all blob reports into one HTML report for every shard of every project (artifact + `playwright-report`). + +To add machines, raise `total` for a project in the `shards` matrix and `shards` in the `tests` matrix to the same +number. Every shard also means 2 more workers hitting the public demo shop, which rate-limits bursts, so 2 per browser +is the current balance (a browser job took about 2 to 2.5 minutes unsharded). + +On CI there are 2 retries, 2 workers per shard, and `forbidOnly`, so a forgotten `test.only` fails the build. Each `tests` job also uploads its JUnit XML (`junit-`) and Playwright's JSON report (`playwright-json-`), which [Automation Coverage & Quality Intelligence](https://github.com/niranjankagri/automation-coverage-quality-intelligence) collects. The JSON keeps every retry attempt: JUnit records a test that passed on a retry as plain passed, so flaky tests are only visible there. diff --git a/playwright.config.ts b/playwright.config.ts index 57a74ee..f7dfd1c 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -23,8 +23,8 @@ export default defineConfig({ timeout: 45_000, expect: { timeout: 10_000 }, - // On CI every job writes a blob report (named after the job, so they don't collide); - // a final job merges them into one HTML report for all browsers. Locally: list + HTML. + // On CI every shard writes a blob report (named after the shard, e.g. report-chromium-1.zip, so they don't collide); + // later jobs merge them per project (JUnit + JSON) and into one HTML report. Locally: list + HTML. reporter: env.isCI ? [ ['github'], diff --git a/src/ci/check-merged-results.mts b/src/ci/check-merged-results.mts new file mode 100644 index 0000000..c5a8afc --- /dev/null +++ b/src/ci/check-merged-results.mts @@ -0,0 +1,56 @@ +/** + * CI gate for one project (api, chromium, firefox or webkit) after its shards have run. + * + * The shard jobs only upload blob reports. The "Tests ()" job merges them into one + * JSON report and runs this script, which fails the job when: + * 1. a shard is missing (it crashed or was cancelled before uploading its blob report), or + * 2. any test failed on every attempt ("unexpected" in Playwright's JSON stats). + * A test that passed on a retry counts as flaky, not failed, exactly as in an unsharded run. + * + * Usage (Node 24 runs TypeScript files directly): + * node src/ci/check-merged-results.mts + */ +import { existsSync, readFileSync, readdirSync } from 'node:fs'; + +/** The part of Playwright's JSON report this script reads. */ +interface JsonReportStats { + expected: number; + unexpected: number; + flaky: number; + skipped: number; +} + +const [blobDir, expectedShardsText, resultsFile] = process.argv.slice(2); +if (!blobDir || !expectedShardsText || !resultsFile) { + console.error('usage: check-merged-results.mts '); + process.exit(2); +} + +const problems: string[] = []; + +// 1. Every shard uploaded a blob report (one .zip per shard). +const expectedShards = Number(expectedShardsText); +const blobs = existsSync(blobDir) ? readdirSync(blobDir).filter((name) => name.endsWith('.zip')) : []; +console.log(`blob reports: ${blobs.length} of ${expectedShards} (${blobs.join(', ') || 'none'})`); +if (blobs.length !== expectedShards) { + problems.push(`expected ${expectedShards} blob reports, found ${blobs.length}: a shard did not finish`); +} + +// 2. No test failed. Without any blob report there is no results file either. +if (existsSync(resultsFile)) { + const { stats } = JSON.parse(readFileSync(resultsFile, 'utf8')) as { stats: JsonReportStats }; + console.log( + `tests: ${stats.expected} passed, ${stats.flaky} flaky, ${stats.skipped} skipped, ${stats.unexpected} failed`, + ); + if (stats.unexpected > 0) { + problems.push(`${stats.unexpected} test(s) failed (details in the merged HTML report)`); + } +} else { + problems.push(`no merged results at ${resultsFile}`); +} + +if (problems.length > 0) { + for (const problem of problems) console.error(`FAILED: ${problem}`); + process.exit(1); +} +console.log('OK: all shards finished and no test failed'); diff --git a/src/pages/CheckoutPage.ts b/src/pages/CheckoutPage.ts index c06d1c3..656959e 100644 --- a/src/pages/CheckoutPage.ts +++ b/src/pages/CheckoutPage.ts @@ -110,24 +110,47 @@ export class CheckoutPage extends BasePage { await this.proceedFromSignInButton.click(); } - /** Step 3 -> 4 */ + /** + * Step 3 -> 4. After country, postal code and house number the shop calls + * GET /postcode-lookup and fills street, city and state from its answer. + * + * Wait for that answer to be IN the form before going on. The logged-in test user's own + * address (see AuthApi.register) is already in the form, so "Proceed" is enabled at once; + * clicking it before the lookup lands sends that profile address, which the shop then + * rejects at POST /invoices (422 "The city does not belong to the selected country"). + * That happened on slow CI runs only, never locally, where the lookup is always first. + */ @step('Checkout step 3: enter the billing address') async enterBillingAddress(address: BillingAddress): Promise { + // Start listening before typing, so a fast answer cannot be missed. Only the lookup for + // the complete address counts: typing may also trigger lookups for partial input. + const lookup = this.page.waitForResponse( + (response) => + response.url().includes('/postcode-lookup') && + response.url().includes(`postcode=${address.postalCode}`) && + response.url().includes(`house_number=${address.houseNumber}`), + ); await this.countrySelect.selectOption({ label: address.country }); await this.postalCodeInput.fill(address.postalCode); await this.houseNumberInput.fill(address.houseNumber); - // The button only becomes enabled once the form is valid (after the address lookup). + + const response = await lookup; + expect(response.ok(), `postcode lookup failed: ${response.status()}`).toBe(true); + const found = (await response.json()) as { street: string }; + // The street field shows the looked-up street: the form now holds the lookup's address. + await expect(this.streetInput).toHaveValue(found.street); + await expect(this.proceedFromAddressButton).toBeEnabled(); await this.proceedFromAddressButton.click(); } /** * Step 4: choose a payment method, have it checked, then confirm the order. - * Confirm places the order with POST /invoices. The shared demo sometimes rejects a valid - * address there (422 "The city does not belong to the selected country" for 1010 Vienna, AT - * in CI run 40276006707; the retry with the same data passed) and the page shows nothing, - * so the API's answer is checked here: a rejected order fails with the shop's reason - * instead of a 10 s wait for a confirmation that never comes. + * Confirm places the order with POST /invoices. When the shop rejects the order the page + * shows nothing, so the API's answer is checked here: a rejected order fails with the + * shop's reason instead of a 10 s wait for a confirmation that never comes. (That reason is + * how the step 3 race was found: 422 "The city does not belong to the selected country" + * for the profile address, see enterBillingAddress.) */ @step('Checkout step 4: pay with {0}') async payWith(method: PaymentMethod): Promise { diff --git a/tsconfig.json b/tsconfig.json index a16b9b9..2623466 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -14,5 +14,5 @@ "noEmit": true, "types": ["node"] }, - "include": ["src/**/*.ts", "tests/**/*.ts", "playwright.config.ts"] + "include": ["src/**/*.ts", "src/**/*.mts", "tests/**/*.ts", "playwright.config.ts"] }