diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f8edfa..b32764a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,10 +1,17 @@ name: CI -# Push is the trigger that matters: this repo is reviewed off a branch rather -# than through pull requests. The pull_request trigger costs one line and means -# the checks are already wired if that ever changes. +# Pull requests only. An unfiltered `push:` alongside this ran the whole job +# twice for every commit on a PR branch -- once for the push, once for the +# synchronize -- which was the bulk of the CI spend. +# +# The trade-off: nothing runs on the merge commit itself, so the tests never +# see exactly what lands on master. Netlify covers the worst of that, since it +# builds master on merge and a broken build shows up as a failed deploy -- but +# smoke and e2e do not run there. If that gap ever bites, the fix is two lines: +# +# push: +# branches: [master] on: - push: pull_request: concurrency: @@ -67,23 +74,6 @@ jobs: env: BASE_URL: http://localhost:9100/tech-blog/ - # Runs in the Playwright container, not on the runner. Screenshot - # comparison is only meaningful if the baselines were rasterised in the - # same environment: `monospace` is whatever font the OS supplies, and it - # resolves three different ways across a dev machine (DejaVu Sans Mono), - # this runner, and this image (WenQuanYi Zen Hei Mono). Body text hid the - # problem because Open Sans is a webfont; the code blocks did not. - # tests/README.md documents regenerating baselines the same way. - - name: Visual regression - run: | - docker run --rm --network host \ - -v "$PWD/tests":/tests -w /tests \ - --user "$(id -u):$(id -g)" -e HOME=/tmp \ - -e BASE_URL=http://localhost:9100/tech-blog/ \ - -e CI=true \ - mcr.microsoft.com/playwright:v1.62.1-noble \ - npx playwright test --project=desktop --project=tablet --project=mobile - - name: Upload failure artifacts if: failure() uses: actions/upload-artifact@v4 diff --git a/package.json b/package.json index cc5cb74..f847cd7 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,6 @@ "test": "npm --prefix tests test", "test:smoke": "npm --prefix tests run test:smoke", "test:e2e": "npm --prefix tests run test:e2e", - "test:visual": "npm --prefix tests run test:visual", "favIcon16": "sharp -i ./src/images/app-icons/icon.png -o ./static/icons/favicon-16x16.png resize 16", "favIcon32": "sharp -i ./src/images/app-icons/icon.png -o ./static/icons/favicon-32x32.png resize 32", "favIcon96": "sharp -i ./src/images/app-icons/icon.png -o ./static/icons/favicon-96x96.png resize 96", diff --git a/tests/BASELINE.md b/tests/BASELINE.md index d2e6089..8ae1c41 100644 --- a/tests/BASELINE.md +++ b/tests/BASELINE.md @@ -9,7 +9,7 @@ This is the reference the Gatsby 5 migration is checked against. |---|---| | Live URL | https://www.boldare.com/tech-blog/ | | `sitemap.xml` URLs | **148** — 27 posts, 1 page, 118 tag pages, plus `/`, `/search/`, `/tags/` | -| Visual baselines | 18 PNGs — 6 pages × 3 viewports (375 / 768 / 1440) | +| Visual baselines | 18 PNGs — 6 pages × 3 viewports (375 / 768 / 1440). Retired after the upgrade verified 18/18; see tests/README.md | | e2e journeys | 4, all passing against production | ## CMS editorial workflow @@ -49,3 +49,15 @@ steadier, not flakier. was under the old phase-1 plan, since the target is now an explicit `NODE_VERSION = "24"` rather than "record whatever works today". - **GTM container ID** for the analytics swap — needed before C2 can finish. + +## Outcome + +Verified against the final deploy preview on 2026-09-03: + +- 18/18 screenshots matched pre-upgrade production, across 6 pages and 3 viewports. +- 9/9 journeys and widget metrics passed against the deployed site. +- 27 of 28 content pages render structurally identical to production. The + exception is the table of contents on `Dev-and-prod-ready-Docker-setup-for-SPA-app`, + which renders tight where production renders loose: the source has no blank + lines between items, so CommonMark says tight and remark 9 had been lenient. + Left as-is deliberately. diff --git a/tests/README.md b/tests/README.md index 4b4ff94..f2e5d5a 100644 --- a/tests/README.md +++ b/tests/README.md @@ -11,48 +11,31 @@ npm ci npx playwright install chromium ``` -## The three suites +## The two suites | Suite | What it checks | Target | |---|---|---| -| `npm run test:smoke` | Crawls built HTML: every page has title/description/canonical/h1, no broken internal links, no missing images, sitemap + rss present | `../public` on disk | -| `npm run test:visual` | Screenshot diff at 375 / 768 / 1440px | `BASE_URL` | -| `npm run test:e2e` | Reader journeys, Algolia search, `/admin/` boots | `BASE_URL` | +| `npm run test:smoke` | Crawls built HTML: every page has title/description/canonical/h1, no broken internal links, no missing images, sitemap + rss present. Also checks the Algolia query and transformer, which no build without credentials ever exercises | `../public` on disk | +| `npm run test:e2e` | Reader journeys, Algolia search, `/admin/` boots, and the widget metrics MUI 9 changed | `BASE_URL` | -See `BASELINE.md` for what the pre-upgrade state actually was. - -## Visual baselines must be made in the container +## There used to be a screenshot suite -`monospace` is whatever font the operating system supplies, and it resolves three -different ways across a dev machine (DejaVu Sans Mono), a GitHub runner, and the -Playwright image (WenQuanYi Zen Hei Mono). Body text hides this, because Open Sans is a -webfont and is downloaded identically everywhere — code blocks and inline `` do -not, and different glyph widths reflow the line around them. Baselines rasterised on a -host therefore fail in CI for reasons that have nothing to do with the site. +Six pages x three viewports, diffed against baselines captured from production +before the Gatsby 5 upgrade. It answered one question — "does the upgraded site +still look like the site as it was?" — confirmed 18/18 against the final deploy +preview, and was removed once that question was settled. Keeping it would have +meant comparing forever against a site that no longer exists. -So capture and comparison both run in `mcr.microsoft.com/playwright`, which is what the -CI job uses too. **Never run `--update-snapshots` outside it** — `npm run -test:visual:update` on a host will produce baselines that only work on that host. +Its durable half lives on in `e2e/widget-metrics.spec.js`. Screenshot diffing +turned out to be poor at exactly the things that broke here: small chrome is a +rounding error on a full-page image, and `toHaveScreenshot`'s own per-pixel +`threshold` (0.2 by default) makes a whole-page shade change invisible. Asserting +computed values instead is deterministic, needs no container, and caught what the +pixels missed. -```bash -# regenerate baselines FROM PRODUCTION. The point of the suite is comparing against the -# pre-upgrade site, so this target stays production even after the upgrade ships. -docker run --rm --network host -v "$PWD":/tests -w /tests \ - --user "$(id -u):$(id -g)" -e HOME=/tmp \ - -e BASE_URL=https://www.boldare.com/tech-blog/ \ - mcr.microsoft.com/playwright:v1.62.1-noble \ - npx playwright test --project=desktop --project=tablet --project=mobile --update-snapshots +See `BASELINE.md` for what the pre-upgrade state actually was. -# compare a local build against them -cd .. && npm run build && npx gatsby serve --prefix-paths -p 9100 & -cd tests && docker run --rm --network host -v "$PWD":/tests -w /tests \ - --user "$(id -u):$(id -g)" -e HOME=/tmp \ - -e BASE_URL=http://localhost:9100/tech-blog/ \ - mcr.microsoft.com/playwright:v1.62.1-noble \ - npx playwright test --project=desktop --project=tablet --project=mobile -``` -`smoke` and `e2e` have no such constraint and run fine on the host. `smoke` needs no server — it reads `public/` straight off disk, so it works the moment `gatsby build` finishes. diff --git a/tests/e2e/widget-metrics.spec.js b/tests/e2e/widget-metrics.spec.js index d47d11c..44155fb 100644 --- a/tests/e2e/widget-metrics.spec.js +++ b/tests/e2e/widget-metrics.spec.js @@ -5,11 +5,16 @@ const { test, expect } = require("@playwright/test"); // still came in under the pixel-ratio threshold, because four small buttons are // a rounding error on a full-page image. These assert the numbers directly. // -// Every expected value here is what production (MUI v3) renders, measured -// against https://www.boldare.com/tech-blog/. MUI 9 changed each of them: -// IconButton padding 12 -> 8, SvgIcon's fixed 24px -> a pxToRem value inflated -// by this theme's typography.fontSize, and Chip's solid fill -> a translucent -// overlay. They are pinned back in src/styles/theme.js. +// Every expected value is the design as it stood before the Gatsby 5 upgrade, +// measured against production while it still ran Gatsby 2. (Once this ships, +// production becomes this code, so "matches production" stops being a +// meaningful check -- these numbers are the record of what it should be.) +// +// MUI 9 changed each of them: IconButton padding 12 -> 8, SvgIcon's fixed 24px +// -> a pxToRem value inflated by this theme's typography.fontSize, and Chip's +// solid fill -> a translucent overlay. They are pinned in src/styles/theme.js. +// +// This file is also what replaced the screenshot suite; see tests/README.md. const POST = "weekly-ai-bites-last-manual-gap-qa-testing/"; diff --git a/tests/package.json b/tests/package.json index 8d632e6..68176b6 100644 --- a/tests/package.json +++ b/tests/package.json @@ -6,10 +6,7 @@ "scripts": { "test": "playwright test", "test:smoke": "playwright test --project=smoke", - "test:e2e": "playwright test --project=e2e", - "test:visual": "playwright test --project=desktop --project=tablet --project=mobile", - "test:visual:update": "docker run --rm --network host -v \"$PWD\":/tests -w /tests --user \"$(id -u):$(id -g)\" -e HOME=/tmp -e BASE_URL=${BASE_URL:-https://www.boldare.com/tech-blog/} mcr.microsoft.com/playwright:v1.62.1-noble npx playwright test --project=desktop --project=tablet --project=mobile --update-snapshots", - "test:visual:docker": "docker run --rm --network host -v \"$PWD\":/tests -w /tests --user \"$(id -u):$(id -g)\" -e HOME=/tmp -e BASE_URL=${BASE_URL:-https://www.boldare.com/tech-blog/} mcr.microsoft.com/playwright:v1.62.1-noble npx playwright test --project=desktop --project=tablet --project=mobile" + "test:e2e": "playwright test --project=e2e" }, "devDependencies": { "@playwright/test": "1.62.1" diff --git a/tests/playwright.config.js b/tests/playwright.config.js index a5fe52f..3e327bb 100644 --- a/tests/playwright.config.js +++ b/tests/playwright.config.js @@ -9,8 +9,6 @@ const chrome = devices["Desktop Chrome"]; module.exports = defineConfig({ testDir: ".", - // Visual diffs against a live site carry some irreducible noise (lazy-loaded - // covers, webfont timing, the Facebook SDK). One retry locally, two in CI. retries: process.env.CI ? 2 : 1, workers: process.env.CI ? 2 : undefined, reporter: process.env.CI ? [["github"], ["list"]] : [["list"]], @@ -19,25 +17,11 @@ module.exports = defineConfig({ trace: "retain-on-failure", screenshot: "only-on-failure", }, - expect: { - toHaveScreenshot: { - // Anti-aliasing and subpixel text rendering differ between machines, so - // some tolerance is needed. 0.02 was too loose: it silently passed a real - // regression where every post thumbnail rendered 90x51 instead of 90x90, - // because the images are a small share of a full-page screenshot. - maxDiffPixelRatio: 0.004, - animations: "disabled", - }, - }, projects: [ // Reads public/ off disk — a browser is used only to parse the HTML, so one // viewport is enough. Running this per-viewport would just triple the work. { name: "smoke", testMatch: /smoke\//, use: { ...chrome } }, { name: "e2e", testMatch: /e2e\//, use: { ...chrome, viewport: { width: 1440, height: 900 } } }, - - { name: "desktop", testMatch: /visual\//, use: { ...chrome, viewport: { width: 1440, height: 900 } } }, - { name: "tablet", testMatch: /visual\//, use: { ...chrome, viewport: { width: 768, height: 1024 } } }, - { name: "mobile", testMatch: /visual\//, use: { ...chrome, viewport: { width: 375, height: 812 } } }, ], }); diff --git a/tests/visual/pages.spec.js b/tests/visual/pages.spec.js deleted file mode 100644 index e478cca..0000000 --- a/tests/visual/pages.spec.js +++ /dev/null @@ -1,94 +0,0 @@ -const { test, expect } = require("@playwright/test"); - -// Baselines are captured from production BEFORE the upgrade, then diffed against a -// local build afterwards. The JSS style objects do not change during the migration, -// so the expectation is near-zero drift: any diff here is a real signal and should -// be explained, not waved through with --update-snapshots. -const PAGES = [ - { name: "home", path: "" }, - { name: "post", path: "quick-import-of-mysql-database-dump/" }, - { name: "page", path: "how-to-contribute/" }, - { name: "tags-index", path: "tags/" }, - { name: "tag", path: "tags/php/" }, - { name: "search", path: "search/", requiresAlgolia: true }, -]; - -// Third-party embeds render differently on every run and are not ours to regress. -const VOLATILE = [ - "#fb-root", - ".fb-comments", - "iframe", - ".ais-Stats", // Algolia result counts and timings -]; - -// src/utils/shared.js drives the navigator between its "featured" and "aside" -// states with chained setTimeouts, and SpringScrollbars runs rebound spring -// physics. Both are still mid-flight at load, which is what made the first -// attempt at these baselines differ by a deterministic 8%. Kill animation and -// let the timer chain finish before looking at the page. -const SETTLE_MS = 2500; - -async function freezeAndSettle(page) { - // Netlify injects a deploy-preview collaboration drawer (an iframe from - // app.netlify.com) that exists on previews and nowhere else. Masking it would - // paint a block the production baseline does not have, so remove it outright. - await page.evaluate(() => { - document - .querySelectorAll('iframe[src*="app.netlify.com"], iframe[src*="netlify.com/cdp"]') - .forEach(el => el.remove()); - }); - - await page.addStyleTag({ - content: `*, *::before, *::after { - animation-duration: 0s !important; - animation-delay: 0s !important; - transition-duration: 0s !important; - transition-delay: 0s !important; - caret-color: transparent !important; - }`, - }); - - // react-lazyload only mounts cover images once they scroll into view. - await page.evaluate(async () => { - await new Promise(resolve => { - let y = 0; - const step = () => { - window.scrollTo(0, y); - y += window.innerHeight; - if (y < document.body.scrollHeight) requestAnimationFrame(step); - else { - window.scrollTo(0, 0); - resolve(); - } - }; - step(); - }); - }); - - await page.evaluate(() => document.fonts.ready); - await page.waitForLoadState("networkidle").catch(() => {}); - await page.waitForTimeout(SETTLE_MS); -} - -for (const { name, path: p, requiresAlgolia } of PAGES) { - test(`${name} matches baseline`, async ({ page }) => { - await page.goto(p, { waitUntil: "networkidle" }); - - // gatsby-config.js only registers Algolia when the credentials are present - // in the build environment, so a local build without them renders no search - // widget and the comparison would be against a page that cannot exist here. - if (requiresAlgolia) { - const box = page.locator(".ais-SearchBox, input[type='search']").first(); - if (!(await box.isVisible().catch(() => false))) { - test.skip(true, "Search widget absent — this build had no ALGOLIA_* credentials"); - } - } - - await freezeAndSettle(page); - - await expect(page).toHaveScreenshot(`${name}.png`, { - fullPage: true, - mask: VOLATILE.map(sel => page.locator(sel)), - }); - }); -} diff --git a/tests/visual/pages.spec.js-snapshots/home-desktop-linux.png b/tests/visual/pages.spec.js-snapshots/home-desktop-linux.png deleted file mode 100644 index 538eb56..0000000 Binary files a/tests/visual/pages.spec.js-snapshots/home-desktop-linux.png and /dev/null differ diff --git a/tests/visual/pages.spec.js-snapshots/home-mobile-linux.png b/tests/visual/pages.spec.js-snapshots/home-mobile-linux.png deleted file mode 100644 index 0b6a125..0000000 Binary files a/tests/visual/pages.spec.js-snapshots/home-mobile-linux.png and /dev/null differ diff --git a/tests/visual/pages.spec.js-snapshots/home-tablet-linux.png b/tests/visual/pages.spec.js-snapshots/home-tablet-linux.png deleted file mode 100644 index ee7f887..0000000 Binary files a/tests/visual/pages.spec.js-snapshots/home-tablet-linux.png and /dev/null differ diff --git a/tests/visual/pages.spec.js-snapshots/page-desktop-linux.png b/tests/visual/pages.spec.js-snapshots/page-desktop-linux.png deleted file mode 100644 index d1d4623..0000000 Binary files a/tests/visual/pages.spec.js-snapshots/page-desktop-linux.png and /dev/null differ diff --git a/tests/visual/pages.spec.js-snapshots/page-mobile-linux.png b/tests/visual/pages.spec.js-snapshots/page-mobile-linux.png deleted file mode 100644 index 273a2ac..0000000 Binary files a/tests/visual/pages.spec.js-snapshots/page-mobile-linux.png and /dev/null differ diff --git a/tests/visual/pages.spec.js-snapshots/page-tablet-linux.png b/tests/visual/pages.spec.js-snapshots/page-tablet-linux.png deleted file mode 100644 index ad3edaf..0000000 Binary files a/tests/visual/pages.spec.js-snapshots/page-tablet-linux.png and /dev/null differ diff --git a/tests/visual/pages.spec.js-snapshots/post-desktop-linux.png b/tests/visual/pages.spec.js-snapshots/post-desktop-linux.png deleted file mode 100644 index 1267cff..0000000 Binary files a/tests/visual/pages.spec.js-snapshots/post-desktop-linux.png and /dev/null differ diff --git a/tests/visual/pages.spec.js-snapshots/post-mobile-linux.png b/tests/visual/pages.spec.js-snapshots/post-mobile-linux.png deleted file mode 100644 index de13fa2..0000000 Binary files a/tests/visual/pages.spec.js-snapshots/post-mobile-linux.png and /dev/null differ diff --git a/tests/visual/pages.spec.js-snapshots/post-tablet-linux.png b/tests/visual/pages.spec.js-snapshots/post-tablet-linux.png deleted file mode 100644 index e921b74..0000000 Binary files a/tests/visual/pages.spec.js-snapshots/post-tablet-linux.png and /dev/null differ diff --git a/tests/visual/pages.spec.js-snapshots/search-desktop-linux.png b/tests/visual/pages.spec.js-snapshots/search-desktop-linux.png deleted file mode 100644 index 77797e8..0000000 Binary files a/tests/visual/pages.spec.js-snapshots/search-desktop-linux.png and /dev/null differ diff --git a/tests/visual/pages.spec.js-snapshots/search-mobile-linux.png b/tests/visual/pages.spec.js-snapshots/search-mobile-linux.png deleted file mode 100644 index 2defc5c..0000000 Binary files a/tests/visual/pages.spec.js-snapshots/search-mobile-linux.png and /dev/null differ diff --git a/tests/visual/pages.spec.js-snapshots/search-tablet-linux.png b/tests/visual/pages.spec.js-snapshots/search-tablet-linux.png deleted file mode 100644 index 2f6776c..0000000 Binary files a/tests/visual/pages.spec.js-snapshots/search-tablet-linux.png and /dev/null differ diff --git a/tests/visual/pages.spec.js-snapshots/tag-desktop-linux.png b/tests/visual/pages.spec.js-snapshots/tag-desktop-linux.png deleted file mode 100644 index 2b78208..0000000 Binary files a/tests/visual/pages.spec.js-snapshots/tag-desktop-linux.png and /dev/null differ diff --git a/tests/visual/pages.spec.js-snapshots/tag-mobile-linux.png b/tests/visual/pages.spec.js-snapshots/tag-mobile-linux.png deleted file mode 100644 index fef464b..0000000 Binary files a/tests/visual/pages.spec.js-snapshots/tag-mobile-linux.png and /dev/null differ diff --git a/tests/visual/pages.spec.js-snapshots/tag-tablet-linux.png b/tests/visual/pages.spec.js-snapshots/tag-tablet-linux.png deleted file mode 100644 index 137aaa6..0000000 Binary files a/tests/visual/pages.spec.js-snapshots/tag-tablet-linux.png and /dev/null differ diff --git a/tests/visual/pages.spec.js-snapshots/tags-index-desktop-linux.png b/tests/visual/pages.spec.js-snapshots/tags-index-desktop-linux.png deleted file mode 100644 index 5c82c0f..0000000 Binary files a/tests/visual/pages.spec.js-snapshots/tags-index-desktop-linux.png and /dev/null differ diff --git a/tests/visual/pages.spec.js-snapshots/tags-index-mobile-linux.png b/tests/visual/pages.spec.js-snapshots/tags-index-mobile-linux.png deleted file mode 100644 index 102a1fa..0000000 Binary files a/tests/visual/pages.spec.js-snapshots/tags-index-mobile-linux.png and /dev/null differ diff --git a/tests/visual/pages.spec.js-snapshots/tags-index-tablet-linux.png b/tests/visual/pages.spec.js-snapshots/tags-index-tablet-linux.png deleted file mode 100644 index e3a161a..0000000 Binary files a/tests/visual/pages.spec.js-snapshots/tags-index-tablet-linux.png and /dev/null differ