Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 69 additions & 13 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Continuous integration: static checks first, then API and UI tests in parallel jobs,
# then one merged HTML report for the whole run.
# Continuous integration: static checks first, then the API and UI tests split into parallel
# shards, then one result per project (api, chromium, firefox, webkit) and one merged HTML report.
#
# Runs on every push and pull request to main, by hand (Actions tab -> Run workflow),
# and every night: the shop under test is a public demo that can change without any
Expand Down Expand Up @@ -36,43 +36,99 @@ jobs:
- run: npm ci
- run: npm run check

tests:
name: Tests (${{ matrix.project }})
# Each browser's UI tests are split across 2 machines with Playwright's --shard; the API
# tests take about a minute, so they stay in one job. More shards also mean more load on the
# public demo shop (2 workers each), so raise `total` only when a project gets slow.
# Keep the shard counts here in step with `shards` in the tests job below.
shards:
name: Shard (${{ matrix.project }} ${{ matrix.shard }}/${{ matrix.total }})
needs: checks # no point starting browsers if the code does not even compile
runs-on: ubuntu-latest
timeout-minutes: 20
strategy:
fail-fast: false # one browser failing should not cancel the others
fail-fast: false # one shard failing should not cancel the others
matrix:
project: [api, chromium, firefox, webkit]
include:
- { project: api, shard: 1, total: 1 }
- { project: chromium, shard: 1, total: 2 }
- { project: chromium, shard: 2, total: 2 }
- { project: firefox, shard: 1, total: 2 }
- { project: firefox, shard: 2, total: 2 }
- { project: webkit, shard: 1, total: 2 }
- { project: webkit, shard: 2, total: 2 }
env:
# Names this job's blob report (see playwright.config.ts).
BLOB_NAME: ${{ matrix.project }}
# Names this shard's blob report, e.g. report-chromium-1.zip (see playwright.config.ts).
BLOB_NAME: ${{ matrix.project }}-${{ matrix.shard }}
steps:
- uses: actions/checkout@v7
- uses: actions/setup-node@v7
with:
node-version: 24
cache: npm
cache: npm # reuses downloaded packages between runs (keyed on package-lock.json)
- run: npm ci

# Only the browser this job needs (with its system libraries); the API job needs none.
- name: Install ${{ matrix.project }}
if: matrix.project != 'api'
run: npx playwright install --with-deps ${{ matrix.project }}

# --shard=1/2 runs the first half of the test files of this project, 2/2 the rest.
- name: Run tests
run: npx playwright test --project=${{ matrix.project }}
run: npx playwright test --project=${{ matrix.project }} --shard=${{ matrix.shard }}/${{ matrix.total }}

# Upload even when tests failed: that is exactly when the report is needed.
- name: Upload blob report
if: ${{ !cancelled() }}
uses: actions/upload-artifact@v7
with:
name: blob-report-${{ matrix.project }}
name: blob-report-${{ matrix.project }}-${{ matrix.shard }}
path: blob-report
retention-days: 3

# One job per project merges that project's shards into a single result, so the pull
# request checks (and the "Protect main" ruleset) still see Tests (api), Tests (chromium),
# Tests (firefox) and Tests (webkit), and the artifacts keep their unsharded names.
tests:
name: Tests (${{ matrix.project }})
needs: shards
if: ${{ !cancelled() }} # also runs when a shard failed: this job reports it
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
include:
- { project: api, shards: 1 }
- { project: chromium, shards: 2 }
- { project: firefox, shards: 2 }
- { project: webkit, shards: 2 }
steps:
- uses: actions/checkout@v7
- uses: actions/setup-node@v7
with:
node-version: 24
cache: npm
- run: npm ci

- name: Download the blob reports of ${{ matrix.project }}
uses: actions/download-artifact@v8
with:
path: blob-reports
pattern: blob-report-${{ matrix.project }}-*
merge-multiple: true

# The same JUnit XML and JSON files one unsharded job would write (test names, retries,
# flaky tests unchanged). Skipped when no shard uploaded anything; the check below says so.
- name: Merge the shards
if: ${{ hashFiles('blob-reports/*.zip') != '' }}
env:
PLAYWRIGHT_JUNIT_OUTPUT_FILE: test-results/junit.xml
PLAYWRIGHT_JSON_OUTPUT_FILE: test-results/results.json
run: npx playwright merge-reports --reporter junit,json ./blob-reports

# Fails this job when a shard is missing or a test failed (see the script's comment).
- name: Check the merged results
run: node src/ci/check-merged-results.mts blob-reports ${{ matrix.shards }} test-results/results.json

# JUnit XML only, as its own artifact: the CI results collection of
# automation-coverage-quality-intelligence imports it (config/producers.toml there)
- name: Keep the JUnit results
Expand All @@ -97,7 +153,7 @@ jobs:

report:
name: Merge HTML report
needs: tests
needs: [shards, tests]
if: ${{ !cancelled() }}
runs-on: ubuntu-latest
steps:
Expand All @@ -115,7 +171,7 @@ jobs:
pattern: blob-report-*
merge-multiple: true

# One HTML report with every project (api, chromium, firefox, webkit) of this run.
# One HTML report with every shard of every project (api, chromium, firefox, webkit) of this run.
- name: Merge into one HTML report
run: npx playwright merge-reports --reporter html ./all-blob-reports

Expand Down
36 changes: 23 additions & 13 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,14 @@ a demo web shop built for testers (Angular front end + REST API), written with

## Tech stack

| | |
| ------------- | ---------------------------------------------------------------------------------------------------------- |
| Test runner | Playwright Test 1.64 (TypeScript, no extra compiler step) |
| Language | TypeScript 6.0, strict mode |
| API contracts | zod 4 schemas, which also provide the TypeScript types |
| Static checks | `tsc`, ESLint (typescript-eslint, type-aware + eslint-plugin-playwright), Prettier |
| CI | GitHub Actions: checks, then 4 parallel jobs (api, chromium, firefox, webkit), then one merged HTML report |
| Node | 20.12 or newer (CI uses 24) |
| | |
| ------------- | --------------------------------------------------------------------------------------------------------------------------------- |
| Test runner | Playwright Test 1.64 (TypeScript, no extra compiler step) |
| Language | TypeScript 6.0, strict mode |
| API contracts | zod 4 schemas, which also provide the TypeScript types |
| Static checks | `tsc`, ESLint (typescript-eslint, type-aware + eslint-plugin-playwright), Prettier |
| CI | GitHub Actions: checks, then 7 parallel shards (`--shard`; api 1, each browser 2), one result per project, one merged HTML report |
| Node | 20.12 or newer (CI uses 24) |

## Quick start

Expand Down Expand Up @@ -178,11 +178,21 @@ snapshots, network and console) and a page snapshot (`error-context.md`). On a r
(the demo can change without a commit here):

1. **checks**: `npm run check` (typecheck, ESLint, Prettier). Nothing else starts if this fails.
2. **tests**: four parallel jobs, `api`, `chromium`, `firefox`, `webkit`; each installs only its own browser and uploads
a blob report, also when tests fail.
3. **report**: merges the blob reports into one HTML report for all four projects (artifact `playwright-report`).

On CI there are 2 retries, 2 workers per job, and `forbidOnly`, so a forgotten `test.only` fails the build. Each test
2. **shards**: seven parallel jobs. The API tests run in one job; the UI tests of each browser are split across two
machines with Playwright's `--shard=1/2` and `--shard=2/2`. Each job installs only its own browser and uploads a blob
report, also when tests fail.
3. **tests**: one job per project (`Tests (api)`, `Tests (chromium)`, ...) merges that project's shards with
`playwright merge-reports` into the same JUnit XML and JSON report an unsharded run writes, then
[`src/ci/check-merged-results.mts`](src/ci/check-merged-results.mts) fails the job if a shard is missing or a test
failed. So the pull request checks and the artifact names did not change when sharding came in.
4. **report**: merges all blob reports into one HTML report for every shard of every project (artifact
`playwright-report`).

To add machines, raise `total` for a project in the `shards` matrix and `shards` in the `tests` matrix to the same
number. Every shard also means 2 more workers hitting the public demo shop, which rate-limits bursts, so 2 per browser
is the current balance (a browser job took about 2 to 2.5 minutes unsharded).

On CI there are 2 retries, 2 workers per shard, and `forbidOnly`, so a forgotten `test.only` fails the build. Each `tests`
job also uploads its JUnit XML (`junit-<project>`) and Playwright's JSON report (`playwright-json-<project>`), which
[Automation Coverage & Quality Intelligence](https://github.com/niranjankagri/automation-coverage-quality-intelligence) collects. The JSON keeps every
retry attempt: JUnit records a test that passed on a retry as plain passed, so flaky tests are only visible there.
Expand Down
4 changes: 2 additions & 2 deletions playwright.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@ export default defineConfig({
timeout: 45_000,
expect: { timeout: 10_000 },

// On CI every job writes a blob report (named after the job, so they don't collide);
// a final job merges them into one HTML report for all browsers. Locally: list + HTML.
// On CI every shard writes a blob report (named after the shard, e.g. report-chromium-1.zip, so they don't collide);
// later jobs merge them per project (JUnit + JSON) and into one HTML report. Locally: list + HTML.
reporter: env.isCI
? [
['github'],
Expand Down
56 changes: 56 additions & 0 deletions src/ci/check-merged-results.mts
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
/**
* CI gate for one project (api, chromium, firefox or webkit) after its shards have run.
*
* The shard jobs only upload blob reports. The "Tests (<project>)" job merges them into one
* JSON report and runs this script, which fails the job when:
* 1. a shard is missing (it crashed or was cancelled before uploading its blob report), or
* 2. any test failed on every attempt ("unexpected" in Playwright's JSON stats).
* A test that passed on a retry counts as flaky, not failed, exactly as in an unsharded run.
*
* Usage (Node 24 runs TypeScript files directly):
* node src/ci/check-merged-results.mts <blob-report-dir> <expected-shards> <results.json>
*/
import { existsSync, readFileSync, readdirSync } from 'node:fs';

/** The part of Playwright's JSON report this script reads. */
interface JsonReportStats {
expected: number;
unexpected: number;
flaky: number;
skipped: number;
}

const [blobDir, expectedShardsText, resultsFile] = process.argv.slice(2);
if (!blobDir || !expectedShardsText || !resultsFile) {
console.error('usage: check-merged-results.mts <blob-report-dir> <expected-shards> <results.json>');
process.exit(2);
}

const problems: string[] = [];

// 1. Every shard uploaded a blob report (one .zip per shard).
const expectedShards = Number(expectedShardsText);
const blobs = existsSync(blobDir) ? readdirSync(blobDir).filter((name) => name.endsWith('.zip')) : [];
console.log(`blob reports: ${blobs.length} of ${expectedShards} (${blobs.join(', ') || 'none'})`);
if (blobs.length !== expectedShards) {
problems.push(`expected ${expectedShards} blob reports, found ${blobs.length}: a shard did not finish`);
}

// 2. No test failed. Without any blob report there is no results file either.
if (existsSync(resultsFile)) {
const { stats } = JSON.parse(readFileSync(resultsFile, 'utf8')) as { stats: JsonReportStats };
console.log(
`tests: ${stats.expected} passed, ${stats.flaky} flaky, ${stats.skipped} skipped, ${stats.unexpected} failed`,
);
if (stats.unexpected > 0) {
problems.push(`${stats.unexpected} test(s) failed (details in the merged HTML report)`);
}
} else {
problems.push(`no merged results at ${resultsFile}`);
}

if (problems.length > 0) {
for (const problem of problems) console.error(`FAILED: ${problem}`);
process.exit(1);
}
console.log('OK: all shards finished and no test failed');
37 changes: 30 additions & 7 deletions src/pages/CheckoutPage.ts
Original file line number Diff line number Diff line change
Expand Up @@ -110,24 +110,47 @@ export class CheckoutPage extends BasePage {
await this.proceedFromSignInButton.click();
}

/** Step 3 -> 4 */
/**
* Step 3 -> 4. After country, postal code and house number the shop calls
* GET /postcode-lookup and fills street, city and state from its answer.
*
* Wait for that answer to be IN the form before going on. The logged-in test user's own
* address (see AuthApi.register) is already in the form, so "Proceed" is enabled at once;
* clicking it before the lookup lands sends that profile address, which the shop then
* rejects at POST /invoices (422 "The city does not belong to the selected country").
* That happened on slow CI runs only, never locally, where the lookup is always first.
*/
@step('Checkout step 3: enter the billing address')
async enterBillingAddress(address: BillingAddress): Promise<void> {
// Start listening before typing, so a fast answer cannot be missed. Only the lookup for
// the complete address counts: typing may also trigger lookups for partial input.
const lookup = this.page.waitForResponse(
(response) =>
response.url().includes('/postcode-lookup') &&
response.url().includes(`postcode=${address.postalCode}`) &&
response.url().includes(`house_number=${address.houseNumber}`),
);
await this.countrySelect.selectOption({ label: address.country });
await this.postalCodeInput.fill(address.postalCode);
await this.houseNumberInput.fill(address.houseNumber);
// The button only becomes enabled once the form is valid (after the address lookup).

const response = await lookup;
expect(response.ok(), `postcode lookup failed: ${response.status()}`).toBe(true);
const found = (await response.json()) as { street: string };
// The street field shows the looked-up street: the form now holds the lookup's address.
await expect(this.streetInput).toHaveValue(found.street);

await expect(this.proceedFromAddressButton).toBeEnabled();
await this.proceedFromAddressButton.click();
}

/**
* Step 4: choose a payment method, have it checked, then confirm the order.
* Confirm places the order with POST /invoices. The shared demo sometimes rejects a valid
* address there (422 "The city does not belong to the selected country" for 1010 Vienna, AT
* in CI run 40276006707; the retry with the same data passed) and the page shows nothing,
* so the API's answer is checked here: a rejected order fails with the shop's reason
* instead of a 10 s wait for a confirmation that never comes.
* Confirm places the order with POST /invoices. When the shop rejects the order the page
* shows nothing, so the API's answer is checked here: a rejected order fails with the
* shop's reason instead of a 10 s wait for a confirmation that never comes. (That reason is
* how the step 3 race was found: 422 "The city does not belong to the selected country"
* for the profile address, see enterBillingAddress.)
*/
@step('Checkout step 4: pay with {0}')
async payWith(method: PaymentMethod): Promise<void> {
Expand Down
2 changes: 1 addition & 1 deletion tsconfig.json
Original file line number Diff line number Diff line change
Expand Up @@ -14,5 +14,5 @@
"noEmit": true,
"types": ["node"]
},
"include": ["src/**/*.ts", "tests/**/*.ts", "playwright.config.ts"]
"include": ["src/**/*.ts", "src/**/*.mts", "tests/**/*.ts", "playwright.config.ts"]
}
Loading