From f5698272d832135d3b5a6c7928fbaeb2efc30958 Mon Sep 17 00:00:00 2001 From: nstarman Date: Tue, 6 Oct 2026 16:37:48 -0400 Subject: [PATCH] Check the browser-free cards against Chrome weekly, on request and on a label tests/cardlayout.differential.test.js has fast-check generate a width, a title, text, a role and a subset of a package's buttons with labels, lays each card out in Chrome (the real card CSS, on the real embed page) and with cardlayout.js, and compares them; a difference is shrunk to the smallest card that shows it. It needs a browser and the dev server, so it only runs when CARD_FUZZ is set (npx vitest run tests/cardlayout.differential.test.js). .github/workflows/card-layout-fuzz.yml runs it on Mondays, from Actions with a number of cards and an optional seed and path to replay a failure, and when a pull request is given the fuzz-cards label (taken off again after, so adding it again runs it again). A failure is in the step summary and kept as an artifact. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/card-layout-fuzz.yml | 109 +++++++++++++++++ package-lock.json | 30 +++++ package.json | 1 + tests/cardlayout.differential.test.js | 156 +++++++++++++++++++++++++ 4 files changed, 296 insertions(+) create mode 100644 .github/workflows/card-layout-fuzz.yml create mode 100644 tests/cardlayout.differential.test.js diff --git a/.github/workflows/card-layout-fuzz.yml b/.github/workflows/card-layout-fuzz.yml new file mode 100644 index 00000000..3ff95df3 --- /dev/null +++ b/.github/workflows/card-layout-fuzz.yml @@ -0,0 +1,109 @@ +name: Card layout fuzz + +# The browser-free software cards (src/lib/cardlayout.js, which draws the +# README's images) are held to Chrome's own layout of the real card CSS by +# tests/cardlayout.test.js — at a few widths, from a recording. This asks Chrome +# about cards nobody recorded: tests/cardlayout.differential.test.js has +# fast-check generate a width, a title, text, a role and a subset of a +# package's buttons, lays each out in Chrome on the real embed page and with +# cardlayout.js, and compares them. A difference is shrunk to the smallest card +# that shows it, with a seed to replay it. +# +# Slow and needs a browser, so it is not part of CI: it runs weekly, by hand +# (Actions > Run workflow, with a number of cards and optionally a seed to +# replay), and when a pull request is given the fuzz-cards label — the label is +# taken off again when the run ends, so adding it again runs it again. + +on: + schedule: + - cron: "0 5 * * 1" # 05:00 UTC on Mondays + workflow_dispatch: + inputs: + runs: + description: "How many cards to try" + default: "500" + seed: + description: "A seed to replay a failure (from its report); empty for a new one" + default: "" + path: + description: "The failure's path, with the seed, to replay it directly" + default: "" + pull_request: + types: [labeled] + +permissions: + contents: read + pull-requests: write # to take the label off + +concurrency: + group: card-layout-fuzz-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + fuzz: + name: fuzz + # On a pull request, only the one label. + if: github.event_name != 'pull_request' || github.event.label.name == 'fuzz-cards' + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + FUZZ_SITE: http://localhost:4321 + FC_RUNS: ${{ inputs.runs || (github.event_name == 'schedule' && '1000') || '300' }} + FC_SEED: ${{ inputs.seed }} + FC_PATH: ${{ inputs.path }} + FUZZ_OUT: fuzz-report + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 + with: + node-version: "22" + cache: npm + - run: npm ci + - name: Install the browser the cards are laid out in + run: npx playwright install --with-deps chromium + + # The test imports the site's own modules into the page, so it needs the + # dev server, not a build. + - name: Start the site + run: | + nohup npx astro dev --port 4321 > dev.log 2>&1 & + for i in $(seq 1 60); do + curl -sf "$FUZZ_SITE/software/" > /dev/null && exit 0 + sleep 2 + done + cat dev.log + echo "The site did not start." >&2 + exit 1 + + - name: Compare the cards with Chrome's + run: CARD_FUZZ=1 npx vitest run tests/cardlayout.differential.test.js + + - name: Report a difference + if: failure() + run: | + { + echo "### Card layout differs from Chrome's" + echo + echo "Seed and path are in the report: replay with the workflow's \`seed\` and \`path\` inputs, or" + echo '`CARD_FUZZ=1 FC_SEED=… FC_PATH=… npx vitest run tests/cardlayout.differential.test.js`.' + echo + echo '```' + cat fuzz-report/counterexample.txt 2>/dev/null || echo "(no report written: see the log)" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - uses: actions/upload-artifact@v7 + if: failure() + with: + name: fuzz-report + path: | + fuzz-report + dev.log + retention-days: 30 + + # Taken off whatever the result, so that adding it again runs it again. + - name: Take the label off + if: always() && github.event_name == 'pull_request' + env: + GH_TOKEN: ${{ github.token }} + run: gh pr edit "${{ github.event.pull_request.number }}" --repo "${{ github.repository }}" --remove-label fuzz-cards diff --git a/package-lock.json b/package-lock.json index 7d3f41d2..b3b5dc5b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,6 +17,7 @@ "ajv": "^8.17.1", "fast-check": "^4.10.2", "harfbuzzjs": "^1.6.3", + "playwright": "^1.63.0", "sharp": "^0.35.5", "temml": "^0.13.5", "vitest": "^5.0.2" @@ -3579,6 +3580,35 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/playwright": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.63.0.tgz", + "integrity": "sha512-+7ziBLidS4NaNCdt57SUDT+wYmmd5fmiQejUic/kb+YsYSCPyOOE9sebzMjNmQrsnNpDJqd4WHvV/8lfKfUDUg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.63.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/playwright-core": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.63.0.tgz", + "integrity": "sha512-rYCsBF/M5HjUch52bbtVONEFjv6Xu8sm8h72dNlR5bzIE1fvC/bxgspzkjSfU+MweEMmPM8KJebG6nnyxo5mCg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/postcss": { "version": "8.5.26", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.26.tgz", diff --git a/package.json b/package.json index e87ad0a9..ef9a4644 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "ajv": "^8.17.1", "fast-check": "^4.10.2", "harfbuzzjs": "^1.6.3", + "playwright": "^1.63.0", "sharp": "^0.35.5", "temml": "^0.13.5", "vitest": "^5.0.2" diff --git a/tests/cardlayout.differential.test.js b/tests/cardlayout.differential.test.js new file mode 100644 index 00000000..1935e863 --- /dev/null +++ b/tests/cardlayout.differential.test.js @@ -0,0 +1,156 @@ +// The browser-free card (src/lib/cardlayout.js) against Chrome, over generated +// data. tests/cardlayout.test.js holds it to a recording of Chrome at a few +// widths; tests/cardlayout.property.test.js holds it to what must be true of +// any card; this one asks Chrome itself, for cards nobody recorded: a random +// width, a random title, text and role, a random subset of the package's +// buttons with random labels — laid out by the real card CSS on the real embed +// page, and by cardlayout.js, and compared. A difference is shrunk to the +// smallest card that shows it. +// +// It needs a browser and the site, so it runs only when asked: +// +// npx astro dev --port 4321 & +// CARD_FUZZ=1 npx vitest run tests/cardlayout.differential.test.js +// +// FUZZ_SITE (default http://localhost:4321) is where the site is, FC_RUNS how +// many cards to try (100), FUZZ_MINUTES the longest to run (20), FC_SEED and +// FC_PATH replay a failure, FUZZ_OUT a +// directory for it as JSON. CI runs it weekly, on request and when a pull +// request is labelled fuzz-cards: .github/workflows/card-layout-fuzz.yml. + +import fs from 'node:fs'; +import fc from 'fast-check'; +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; +import { softwareInput, softwareModel } from '../src/lib/cardlayout.js'; +import { measure } from '../src/lib/textmeasure.js'; +import { softwareCards, THEMES } from '../src/lib/softwarecards.js'; + +const enabled = !!process.env.CARD_FUZZ; +const site = process.env.FUZZ_SITE ?? 'http://localhost:4321'; +const numRuns = +(process.env.FC_RUNS ?? 100); +const seed = process.env.FC_SEED ? +process.env.FC_SEED : undefined; +const path = process.env.FC_PATH; +const minutes = +(process.env.FUZZ_MINUTES ?? 20); +vi.setConfig({ testTimeout: (minutes + 5) * 60000 }); + +// ---- what is generated: as in cardlayout.property.test.js, a card of a package ---- + +const word = fc.oneof( + { weight: 8, arbitrary: fc.stringMatching(/^[A-Za-z]{1,12}$/) }, + { weight: 2, arbitrary: fc.stringMatching(/^[A-Za-z]{1,6}-[A-Za-z]{1,6}$/) }, + { weight: 1, arbitrary: fc.stringMatching(/^[A-Za-z]{1,6}-[0-9]{1,3}$/) }, + { weight: 1, arbitrary: fc.stringMatching(/^[A-Za-z]+[.,;:!?'")(]$/) }, + { weight: 1, arbitrary: fc.constantFrom('—', '·', '&', '', 'a&b', '"quoted"', "it's", 'naïve', 'Zürich', 'ß', '5–10', 'x/y', 'e.g.', '3.14') }, + { weight: 1, arbitrary: fc.stringMatching(/^[A-Za-z]{25,45}$/) }, +); +const sentence = fc.array(word, { minLength: 1, maxLength: 40 }).map((w) => w.join(' ')); +const label = fc.oneof(fc.stringMatching(/^[0-9]{4}$/), fc.stringMatching(/^[0-9]{1,3}(\.[0-9])?k?$/)); +const cards = fc.record({ + id: fc.constantFrom(...softwareCards.map((c) => c.input.id)), + width: fc.integer({ min: 160, max: 900 }), + theme: fc.constantFrom(...THEMES), + title: fc.option(fc.stringMatching(/^[A-Za-z_][A-Za-z0-9_-]{0,30}$/), { nil: null }), + text: fc.option(sentence, { nil: null }), + role: fc.option(sentence, { nil: null }), + // The first button always stays: a card with none is another card (no box for them), which the unit tests have. + keep: fc.array(fc.boolean(), { minLength: 11, maxLength: 11 }).map((k) => [true, ...k]), + labels: fc.array(label, { minLength: 12, maxLength: 12 }), +}); + +/** The input the page's card has, after the same edits are made to it. */ +function edited(input, c) { + let k = 0; + const links = input.links.flatMap((l, i) => { + if (!c.keep[i]) return []; + const labelled = l.year != null || l.count != null; + const text = labelled ? c.labels[k] : null; + k += 1; + return [{ ...l, ...(l.year != null ? { year: text } : {}), ...(l.count != null ? { count: text } : {}) }]; + }); + return { ...input, title: c.title ?? input.title, text: c.text ?? input.text, role: input.role && c.role ? c.role : input.role, links }; +} + +/** The same edits, made to the page. */ +const edit = ({ title, text, role, keep, labels }) => { + const card = document.querySelector('.card'); + if (title != null) for (const t of card.querySelectorAll('.c-name .c-t')) t.textContent = title; + if (text != null) card.querySelector('.c-det').textContent = text; + if (role != null) { const r = card.querySelector('.c-role'); if (r) r.textContent = role; } + let k = 0; + [...card.querySelectorAll('.c-foot .btns > li[data-rel]')].forEach((li, i) => { + if (!keep[i]) { li.remove(); return; } + const y = li.querySelector('.iconyear'); + if (y) y.textContent = labels[k]; + k += 1; + }); +}; + +/** How the two cards differ, as lines: nothing if they do not. */ +function differences(got, want) { + const out = []; + const near = (a, b, tol) => Math.abs(a - b) <= tol; + if (!near(got.h, want.h, 0.5)) out.push(`card height ${got.h.toFixed(2)}, Chrome ${want.h.toFixed(2)}`); + if (got.ops.length !== want.ops.length) { + out.push(`${got.ops.length} ops, Chrome ${want.ops.length}`); + const txt = (m) => m.ops.filter((o) => o.k === 'text').map((o) => o.s); + out.push(` text ${JSON.stringify(txt(got))}`, ` Chrome ${JSON.stringify(txt(want))}`); + return out; + } + got.ops.forEach((o, i) => { + const w = want.ops[i]; + const at = `op ${i} ${o.k}${o.s ? ` ${JSON.stringify(o.s)}` : ''}`; + if (o.k !== w.k) return out.push(`${at}: Chrome has a ${w.k}`); + for (const [k, tol] of [['x', 0.3], ['y', 0.3], ['w', o.k === 'text' ? 0.75 : 0.3], ['h', 0.3]]) if (!near(o[k], w[k], tol)) out.push(`${at}: ${k} ${o[k].toFixed(2)}, Chrome ${w[k].toFixed(2)}`); + for (const k of ['s', 'font', 'weight', 'color', 'fill', 'stroke', 'href', 'svg']) if (o[k] !== w[k]) out.push(`${at}: ${k} ${JSON.stringify(o[k])}, Chrome ${JSON.stringify(w[k])}`); + }); + return out; +} + +describe.skipIf(!enabled)('against Chrome, over generated cards', () => { + let browser, page; + const real = new Map(); + + beforeAll(async () => { + const { chromium } = await import('playwright'); + browser = await chromium.launch(); + // 1280 wide, where the root font size is the 16px cardlayout.js is written for. + page = await browser.newPage({ viewport: { width: 1280, height: 900 } }); + await page.goto(`${site}/software/`); + for (const { input: { id } } of softwareCards) { + // Page code as text: vitest rewrites an import() in a function, which the page could not run. + real.set(id, await page.evaluate(`(async () => { + const { items } = await import('/src/lib/data.js'); + const { softwareInput } = await import('/src/lib/cardlayout.js'); + return softwareInput(items.find((i) => i.id === ${JSON.stringify(id)})); + })()`)); + } + }, 120000); + afterAll(async () => { await browser?.close(); }); + + it('lays a card out where Chrome does', async () => { + const slugs = Object.fromEntries(softwareCards.map((c) => [c.input.id, c.slug])); + const property = fc.asyncProperty(cards, async (c) => { + const slug = slugs[c.id].replace(/^size:\d+:/, `size:${c.width}:`); + const input = edited(real.get(c.id), c); + const want = softwareModel(input, { slug, theme: c.theme, measure }); + await page.goto(`${site}/embed/${c.id}/?card=${encodeURIComponent(slug)}&theme=${c.theme}`, { waitUntil: 'networkidle' }); + await page.evaluate(`(${edit.toString()})(${JSON.stringify(c)})`); + const got = await page.evaluate(`(async () => { + const { measureCard } = await import('/src/lib/cardpdf.js'); + await document.fonts.ready; + return measureCard(document.querySelector('.card'), { title: 'x', site: 'https://nstarkman.space' }).model; + })()`); + const diff = differences(want, got); + if (diff.length) throw new Error(`${slug} (${c.theme})\n${diff.slice(0, 12).join('\n')}`); + }); + try { + // A time limit, shrinking included: a run that cannot finish is a result too. + await fc.assert(property, { numRuns, seed, path, interruptAfterTimeLimit: minutes * 60000, markInterruptAsFailure: false }); + } catch (e) { + // A failure, as a file a workflow can keep. + if (process.env.FUZZ_OUT) { fs.mkdirSync(process.env.FUZZ_OUT, { recursive: true }); fs.writeFileSync(`${process.env.FUZZ_OUT}/counterexample.txt`, String(e.message)); } + throw e; + } + expect(real.size).toBeGreaterThan(0); + }); +});