From 46149d962c6348d4b5aca84508772ff0ca44bdc3 Mon Sep 17 00:00:00 2001 From: niranjankagri Date: Wed, 4 Nov 2026 10:45:46 +0530 Subject: [PATCH 1/2] CI: shard the UI tests with --shard and merge the blob reports Each browser's UI tests now run on 2 machines (--shard=1/2, 2/2), the API tests stay in one job: 7 shard jobs instead of 4 project jobs. A Tests () job per project merges that project's blob reports into the same JUnit XML and JSON report an unsharded run wrote, so the ruleset's required checks and the artifacts Quality Intelligence imports keep their names. src/ci/check-merged-results.mts fails that job when a shard is missing or a test failed. The final job still merges all blob reports into one HTML report. Checked locally: chromium as 2 shards merged -> the same 24 test keys as the unsharded main run 41627424324, retry attempts kept (TC-116 flaky). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 82 +++++++++++++++++++++++++++------ README.md | 36 +++++++++------ playwright.config.ts | 4 +- src/ci/check-merged-results.mts | 56 ++++++++++++++++++++++ tsconfig.json | 2 +- 5 files changed, 151 insertions(+), 29 deletions(-) create mode 100644 src/ci/check-merged-results.mts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a150fe..2a7c261 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,5 @@ -# Continuous integration: static checks first, then API and UI tests in parallel jobs, -# then one merged HTML report for the whole run. +# Continuous integration: static checks first, then the API and UI tests split into parallel +# shards, then one result per project (api, chromium, firefox, webkit) and one merged HTML report. # # Runs on every push and pull request to main, by hand (Actions tab -> Run workflow), # and every night: the shop under test is a public demo that can change without any @@ -36,24 +36,35 @@ jobs: - run: npm ci - run: npm run check - tests: - name: Tests (${{ matrix.project }}) + # Each browser's UI tests are split across 2 machines with Playwright's --shard; the API + # tests take about a minute, so they stay in one job. More shards also mean more load on the + # public demo shop (2 workers each), so raise `total` only when a project gets slow. + # Keep the shard counts here in step with `shards` in the tests job below. + shards: + name: Shard (${{ matrix.project }} ${{ matrix.shard }}/${{ matrix.total }}) needs: checks # no point starting browsers if the code does not even compile runs-on: ubuntu-latest timeout-minutes: 20 strategy: - fail-fast: false # one browser failing should not cancel the others + fail-fast: false # one shard failing should not cancel the others matrix: - project: [api, chromium, firefox, webkit] + include: + - { project: api, shard: 1, total: 1 } + - { project: chromium, shard: 1, total: 2 } + - { project: chromium, shard: 2, total: 2 } + - { project: firefox, shard: 1, total: 2 } + - { project: firefox, shard: 2, total: 2 } + - { project: webkit, shard: 1, total: 2 } + - { project: webkit, shard: 2, total: 2 } env: - # Names this job's blob report (see playwright.config.ts). - BLOB_NAME: ${{ matrix.project }} + # Names this shard's blob report, e.g. report-chromium-1.zip (see playwright.config.ts). + BLOB_NAME: ${{ matrix.project }}-${{ matrix.shard }} steps: - uses: actions/checkout@v7 - uses: actions/setup-node@v7 with: node-version: 24 - cache: npm + cache: npm # reuses downloaded packages between runs (keyed on package-lock.json) - run: npm ci # Only the browser this job needs (with its system libraries); the API job needs none. @@ -61,18 +72,63 @@ jobs: if: matrix.project != 'api' run: npx playwright install --with-deps ${{ matrix.project }} + # --shard=1/2 runs the first half of the test files of this project, 2/2 the rest. - name: Run tests - run: npx playwright test --project=${{ matrix.project }} + run: npx playwright test --project=${{ matrix.project }} --shard=${{ matrix.shard }}/${{ matrix.total }} # Upload even when tests failed: that is exactly when the report is needed. - name: Upload blob report if: ${{ !cancelled() }} uses: actions/upload-artifact@v7 with: - name: blob-report-${{ matrix.project }} + name: blob-report-${{ matrix.project }}-${{ matrix.shard }} path: blob-report retention-days: 3 + # One job per project merges that project's shards into a single result, so the pull + # request checks (and the "Protect main" ruleset) still see Tests (api), Tests (chromium), + # Tests (firefox) and Tests (webkit), and the artifacts keep their unsharded names. + tests: + name: Tests (${{ matrix.project }}) + needs: shards + if: ${{ !cancelled() }} # also runs when a shard failed: this job reports it + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - { project: api, shards: 1 } + - { project: chromium, shards: 2 } + - { project: firefox, shards: 2 } + - { project: webkit, shards: 2 } + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 + with: + node-version: 24 + cache: npm + - run: npm ci + + - name: Download the blob reports of ${{ matrix.project }} + uses: actions/download-artifact@v8 + with: + path: blob-reports + pattern: blob-report-${{ matrix.project }}-* + merge-multiple: true + + # The same JUnit XML and JSON files one unsharded job would write (test names, retries, + # flaky tests unchanged). Skipped when no shard uploaded anything; the check below says so. + - name: Merge the shards + if: ${{ hashFiles('blob-reports/*.zip') != '' }} + env: + PLAYWRIGHT_JUNIT_OUTPUT_FILE: test-results/junit.xml + PLAYWRIGHT_JSON_OUTPUT_FILE: test-results/results.json + run: npx playwright merge-reports --reporter junit,json ./blob-reports + + # Fails this job when a shard is missing or a test failed (see the script's comment). + - name: Check the merged results + run: node src/ci/check-merged-results.mts blob-reports ${{ matrix.shards }} test-results/results.json + # JUnit XML only, as its own artifact: the CI results collection of # automation-coverage-quality-intelligence imports it (config/producers.toml there) - name: Keep the JUnit results @@ -97,7 +153,7 @@ jobs: report: name: Merge HTML report - needs: tests + needs: [shards, tests] if: ${{ !cancelled() }} runs-on: ubuntu-latest steps: @@ -115,7 +171,7 @@ jobs: pattern: blob-report-* merge-multiple: true - # One HTML report with every project (api, chromium, firefox, webkit) of this run. + # One HTML report with every shard of every project (api, chromium, firefox, webkit) of this run. - name: Merge into one HTML report run: npx playwright merge-reports --reporter html ./all-blob-reports diff --git a/README.md b/README.md index a8594ee..d8a6a71 100644 --- a/README.md +++ b/README.md @@ -21,14 +21,14 @@ a demo web shop built for testers (Angular front end + REST API), written with ## Tech stack -| | | -| ------------- | ---------------------------------------------------------------------------------------------------------- | -| Test runner | Playwright Test 1.64 (TypeScript, no extra compiler step) | -| Language | TypeScript 6.0, strict mode | -| API contracts | zod 4 schemas, which also provide the TypeScript types | -| Static checks | `tsc`, ESLint (typescript-eslint, type-aware + eslint-plugin-playwright), Prettier | -| CI | GitHub Actions: checks, then 4 parallel jobs (api, chromium, firefox, webkit), then one merged HTML report | -| Node | 20.12 or newer (CI uses 24) | +| | | +| ------------- | --------------------------------------------------------------------------------------------------------------------------------- | +| Test runner | Playwright Test 1.64 (TypeScript, no extra compiler step) | +| Language | TypeScript 6.0, strict mode | +| API contracts | zod 4 schemas, which also provide the TypeScript types | +| Static checks | `tsc`, ESLint (typescript-eslint, type-aware + eslint-plugin-playwright), Prettier | +| CI | GitHub Actions: checks, then 7 parallel shards (`--shard`; api 1, each browser 2), one result per project, one merged HTML report | +| Node | 20.12 or newer (CI uses 24) | ## Quick start @@ -178,11 +178,21 @@ snapshots, network and console) and a page snapshot (`error-context.md`). On a r (the demo can change without a commit here): 1. **checks**: `npm run check` (typecheck, ESLint, Prettier). Nothing else starts if this fails. -2. **tests**: four parallel jobs, `api`, `chromium`, `firefox`, `webkit`; each installs only its own browser and uploads - a blob report, also when tests fail. -3. **report**: merges the blob reports into one HTML report for all four projects (artifact `playwright-report`). - -On CI there are 2 retries, 2 workers per job, and `forbidOnly`, so a forgotten `test.only` fails the build. Each test +2. **shards**: seven parallel jobs. The API tests run in one job; the UI tests of each browser are split across two + machines with Playwright's `--shard=1/2` and `--shard=2/2`. Each job installs only its own browser and uploads a blob + report, also when tests fail. +3. **tests**: one job per project (`Tests (api)`, `Tests (chromium)`, ...) merges that project's shards with + `playwright merge-reports` into the same JUnit XML and JSON report an unsharded run writes, then + [`src/ci/check-merged-results.mts`](src/ci/check-merged-results.mts) fails the job if a shard is missing or a test + failed. So the pull request checks and the artifact names did not change when sharding came in. +4. **report**: merges all blob reports into one HTML report for every shard of every project (artifact + `playwright-report`). + +To add machines, raise `total` for a project in the `shards` matrix and `shards` in the `tests` matrix to the same +number. Every shard also means 2 more workers hitting the public demo shop, which rate-limits bursts, so 2 per browser +is the current balance (a browser job took about 2 to 2.5 minutes unsharded). + +On CI there are 2 retries, 2 workers per shard, and `forbidOnly`, so a forgotten `test.only` fails the build. Each `tests` job also uploads its JUnit XML (`junit-`) and Playwright's JSON report (`playwright-json-`), which [Automation Coverage & Quality Intelligence](https://github.com/niranjankagri/automation-coverage-quality-intelligence) collects. The JSON keeps every retry attempt: JUnit records a test that passed on a retry as plain passed, so flaky tests are only visible there. diff --git a/playwright.config.ts b/playwright.config.ts index 57a74ee..f7dfd1c 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -23,8 +23,8 @@ export default defineConfig({ timeout: 45_000, expect: { timeout: 10_000 }, - // On CI every job writes a blob report (named after the job, so they don't collide); - // a final job merges them into one HTML report for all browsers. Locally: list + HTML. + // On CI every shard writes a blob report (named after the shard, e.g. report-chromium-1.zip, so they don't collide); + // later jobs merge them per project (JUnit + JSON) and into one HTML report. Locally: list + HTML. reporter: env.isCI ? [ ['github'], diff --git a/src/ci/check-merged-results.mts b/src/ci/check-merged-results.mts new file mode 100644 index 0000000..c5a8afc --- /dev/null +++ b/src/ci/check-merged-results.mts @@ -0,0 +1,56 @@ +/** + * CI gate for one project (api, chromium, firefox or webkit) after its shards have run. + * + * The shard jobs only upload blob reports. The "Tests ()" job merges them into one + * JSON report and runs this script, which fails the job when: + * 1. a shard is missing (it crashed or was cancelled before uploading its blob report), or + * 2. any test failed on every attempt ("unexpected" in Playwright's JSON stats). + * A test that passed on a retry counts as flaky, not failed, exactly as in an unsharded run. + * + * Usage (Node 24 runs TypeScript files directly): + * node src/ci/check-merged-results.mts + */ +import { existsSync, readFileSync, readdirSync } from 'node:fs'; + +/** The part of Playwright's JSON report this script reads. */ +interface JsonReportStats { + expected: number; + unexpected: number; + flaky: number; + skipped: number; +} + +const [blobDir, expectedShardsText, resultsFile] = process.argv.slice(2); +if (!blobDir || !expectedShardsText || !resultsFile) { + console.error('usage: check-merged-results.mts '); + process.exit(2); +} + +const problems: string[] = []; + +// 1. Every shard uploaded a blob report (one .zip per shard). +const expectedShards = Number(expectedShardsText); +const blobs = existsSync(blobDir) ? readdirSync(blobDir).filter((name) => name.endsWith('.zip')) : []; +console.log(`blob reports: ${blobs.length} of ${expectedShards} (${blobs.join(', ') || 'none'})`); +if (blobs.length !== expectedShards) { + problems.push(`expected ${expectedShards} blob reports, found ${blobs.length}: a shard did not finish`); +} + +// 2. No test failed. Without any blob report there is no results file either. +if (existsSync(resultsFile)) { + const { stats } = JSON.parse(readFileSync(resultsFile, 'utf8')) as { stats: JsonReportStats }; + console.log( + `tests: ${stats.expected} passed, ${stats.flaky} flaky, ${stats.skipped} skipped, ${stats.unexpected} failed`, + ); + if (stats.unexpected > 0) { + problems.push(`${stats.unexpected} test(s) failed (details in the merged HTML report)`); + } +} else { + problems.push(`no merged results at ${resultsFile}`); +} + +if (problems.length > 0) { + for (const problem of problems) console.error(`FAILED: ${problem}`); + process.exit(1); +} +console.log('OK: all shards finished and no test failed'); diff --git a/tsconfig.json b/tsconfig.json index a16b9b9..2623466 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -14,5 +14,5 @@ "noEmit": true, "types": ["node"] }, - "include": ["src/**/*.ts", "tests/**/*.ts", "playwright.config.ts"] + "include": ["src/**/*.ts", "src/**/*.mts", "tests/**/*.ts", "playwright.config.ts"] } From 7bca7e629fb62462634c9f97e6becc6332a62751 Mon Sep 17 00:00:00 2001 From: niranjankagri Date: Wed, 4 Nov 2026 11:07:00 +0530 Subject: [PATCH 2/2] Fix TC-116: wait for the postcode lookup before leaving the address step The logged-in test user's profile address is already in the billing form, so Proceed was enabled at once. When GET /postcode-lookup answered late (CI), the order went out with the profile address (Stephansplatz 1, Vienna) and POST /invoices answered 422 "The city does not belong to the selected country". enterBillingAddress now waits for the lookup of the complete address and for its street to be in the form. Reproduced locally by delaying the lookup 3 s: old steps -> the same 422 with the profile address in the body; fixed page object -> order confirmed. Co-Authored-By: Claude Opus 5.5 --- src/pages/CheckoutPage.ts | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/src/pages/CheckoutPage.ts b/src/pages/CheckoutPage.ts index c06d1c3..656959e 100644 --- a/src/pages/CheckoutPage.ts +++ b/src/pages/CheckoutPage.ts @@ -110,24 +110,47 @@ export class CheckoutPage extends BasePage { await this.proceedFromSignInButton.click(); } - /** Step 3 -> 4 */ + /** + * Step 3 -> 4. After country, postal code and house number the shop calls + * GET /postcode-lookup and fills street, city and state from its answer. + * + * Wait for that answer to be IN the form before going on. The logged-in test user's own + * address (see AuthApi.register) is already in the form, so "Proceed" is enabled at once; + * clicking it before the lookup lands sends that profile address, which the shop then + * rejects at POST /invoices (422 "The city does not belong to the selected country"). + * That happened on slow CI runs only, never locally, where the lookup is always first. + */ @step('Checkout step 3: enter the billing address') async enterBillingAddress(address: BillingAddress): Promise { + // Start listening before typing, so a fast answer cannot be missed. Only the lookup for + // the complete address counts: typing may also trigger lookups for partial input. + const lookup = this.page.waitForResponse( + (response) => + response.url().includes('/postcode-lookup') && + response.url().includes(`postcode=${address.postalCode}`) && + response.url().includes(`house_number=${address.houseNumber}`), + ); await this.countrySelect.selectOption({ label: address.country }); await this.postalCodeInput.fill(address.postalCode); await this.houseNumberInput.fill(address.houseNumber); - // The button only becomes enabled once the form is valid (after the address lookup). + + const response = await lookup; + expect(response.ok(), `postcode lookup failed: ${response.status()}`).toBe(true); + const found = (await response.json()) as { street: string }; + // The street field shows the looked-up street: the form now holds the lookup's address. + await expect(this.streetInput).toHaveValue(found.street); + await expect(this.proceedFromAddressButton).toBeEnabled(); await this.proceedFromAddressButton.click(); } /** * Step 4: choose a payment method, have it checked, then confirm the order. - * Confirm places the order with POST /invoices. The shared demo sometimes rejects a valid - * address there (422 "The city does not belong to the selected country" for 1010 Vienna, AT - * in CI run 40276006707; the retry with the same data passed) and the page shows nothing, - * so the API's answer is checked here: a rejected order fails with the shop's reason - * instead of a 10 s wait for a confirmation that never comes. + * Confirm places the order with POST /invoices. When the shop rejects the order the page + * shows nothing, so the API's answer is checked here: a rejected order fails with the + * shop's reason instead of a 10 s wait for a confirmation that never comes. (That reason is + * how the step 3 race was found: 422 "The city does not belong to the selected country" + * for the profile address, see enterBillingAddress.) */ @step('Checkout step 4: pay with {0}') async payWith(method: PaymentMethod): Promise {