diff --git a/.github/workflows/accessibility.yml b/.github/workflows/accessibility.yml index 159c5ff..4916ffe 100644 --- a/.github/workflows/accessibility.yml +++ b/.github/workflows/accessibility.yml @@ -28,7 +28,7 @@ jobs: node-version: 22 # In your own repository this becomes: - # uses: likeBloodMoon/eaa-kit@v0.9.1 + # uses: likeBloodMoon/eaa-kit@v0.10.0 # # An exact release tag. There is deliberately no moving v0 tag to follow: # this is a 0.x package, the flags and the JSON contract can still move diff --git a/.github/workflows/soak.yml b/.github/workflows/soak.yml new file mode 100644 index 0000000..d4326df --- /dev/null +++ b/.github/workflows/soak.yml @@ -0,0 +1,122 @@ +name: Soak + +# Real projects, built from scratch, every night. +# +# The stack fixtures in tests/fixtures/stacks are file layouts, checked on every +# pull request, and they are only as true as the day they were copied. A +# framework's next release can move its output directory or change a manifest +# format without touching any of them. This scaffolds each framework with its +# own official starter, installs it, builds it the way `npx eaa-kit` would, and +# audits it, so a change on the framework's side shows up here within a day, +# and before somebody's first run meets it. +# +# Too slow and too dependent on registries to gate a pull request. A red run +# here is triaged into a fix or an issue, not left standing. + +on: + schedule: + - cron: '17 3 * * *' + workflow_dispatch: + +permissions: + contents: read + +jobs: + stack: + name: ${{ matrix.stack }} on ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + stack: [next, astro, sveltekit, nuxt, docusaurus, vite] + include: + # The one most people meet first, on the other two platforms too. + - os: windows-latest + stack: next + scaffold: npx --yes create-next-app@16 site --yes --use-npm --skip-install + - os: macos-latest + stack: next + scaffold: npx --yes create-next-app@16 site --yes --use-npm --skip-install + - stack: next + scaffold: npx --yes create-next-app@16 site --yes --use-npm --skip-install + - stack: astro + scaffold: npm create --yes astro@4 site -- --template minimal --no-install --no-git --yes + - stack: sveltekit + scaffold: npx --yes sv@0 create site --template minimal --types ts --no-add-ons --no-install + - stack: nuxt + scaffold: npx --yes nuxi@3 init site --template minimal --packageManager npm --gitInit false --no-install + - stack: docusaurus + scaffold: npx --yes create-docusaurus@3 site classic --javascript --package-manager npm --skip-install + - stack: vite + scaffold: npm create --yes vite@7 site -- --template react-ts --no-interactive + # A Vite app is an empty shell until its script runs. + browser: true + + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + + steps: + - uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: pnpm + + - run: pnpm install --frozen-lockfile + + - run: pnpm build + + - name: Scaffold ${{ matrix.stack }} + working-directory: ${{ runner.temp }} + shell: bash + # No terminal, as in CI: every starter here was checked to run + # without asking anything. + run: ${{ matrix.scaffold }} < /dev/null + + - name: Install its dependencies + working-directory: ${{ runner.temp }}/site + shell: bash + run: | + npm install --no-audit --no-fund + if [ "${{ matrix.browser }}" = "true" ]; then + npm install --no-audit --no-fund -D playwright + npx playwright install ${{ runner.os == 'Linux' && '--with-deps' || '' }} chromium + fi + + - name: What detect makes of it + working-directory: ${{ runner.temp }}/site + shell: bash + run: node "$GITHUB_WORKSPACE/dist/cli/index.js" detect --json | tee detect.json + + # Exit 1 means barriers were found, which is a working run. Exit 2 means + # it could not audit the project at all, which is what this job exists + # to catch. + - name: Audit it with no directory, so detection decides + working-directory: ${{ runner.temp }}/site + shell: bash + run: | + set +e + node "$GITHUB_WORKSPACE/dist/cli/index.js" audit ${{ matrix.browser && '--browser' || '' }} \ + --format json --output report.json + code=$? + set -e + echo "exit code $code" + test "$code" -le 1 + node -e ' + const r = require("./report.json") + const pages = r.pages.length + console.log(`${pages} pages, discovery ${r.completeness.discovery}`) + if (pages === 0) process.exit(1) + ' + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: soak-${{ matrix.stack }}-${{ matrix.os }} + path: | + ${{ runner.temp }}/site/detect.json + ${{ runner.temp }}/site/report.json + if-no-files-found: ignore diff --git a/CHANGELOG.md b/CHANGELOG.md index 17acb0c..ebe8858 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,80 @@ move: the JSON report's `schemaVersion` and the baseline file's. Both are bumped a field is removed, renamed, or changes meaning — new fields may appear without one, so consumers must ignore what they do not recognise. +## 0.10.0 — 2026-09-26 + +It finds your site, whatever it is built with. + +### Added + +- **Next.js, in depth.** With no directory, a Next.js app that renders on a server is + built, started with `next start` on a free port, and crawled from the list of pages its + own build manifests give, so a page nothing links to is still audited. `basePath`, the + default locale's unprefixed paths and `trailingSlash` are respected; API routes, error + pages and metadata files are left out. A dynamic route with no prerendered pages is named + as not audited, with the reason. A standalone build is served by its `server.js` when its + static files are in place. `next dev` is never used. +- **Twenty more stacks**: + - apps: Qwik, SolidStart, TanStack Start, Analog, Vue CLI, Parcel, Rsbuild, Rspack and + Ember; + - documentation and static generators: Hexo, MkDocs, Sphinx, mdBook, Zola, Quarto and + Pelican, each with its output directory read out of its config; + - never started, and pointed at `--url`: Drupal, Statamic, and Ghost and Shopify themes. + + Hugo is also found through `config/_default/`. Zola is told from Hugo by what + `config.toml` says, and Hexo from Jekyll by its dependency. +- **Monorepos.** Run from the root of a pnpm, yarn or npm workspace, or a Turborepo, Nx or + Lerna repository, `eaa-kit` finds the packages that are sites. One site is audited as + though the command ran inside it. Several are listed with the command for each. `init` + asks which site to set up and writes the config there. +- **Package managers.** Corepack's `packageManager` field is read first, then the lockfile: + Bun's text `bun.lock`, `deno.lock` and `package-lock.json` as well as pnpm's, yarn's and + Bun's binary one. The lockfile is looked for up to the repository root. Deno runs scripts + as tasks, and `init` writes a Deno or Bun setup step into the workflow. +- **`eaa-kit detect [dir]`** says what an audit here would do, and the evidence for each + part: the framework and what identified it, the package manager and why, the build + output or what would be built or started. It builds, starts and writes nothing. + `--json` prints the same as data. +- **`eaa-kit doctor [dir]`** checks, on one screen, everything the tool needs in a + project. Each problem is followed by the command that fixes it: + - the Node.js version; + - the package manager; + - the site; + - the config; + - a GitHub, GitLab or Bitbucket pipeline; + - the baseline and any expired entries; + - Playwright and Chromium. + + It exits 2 only for what stops an audit from running. +- **A recorded answer for every stack.** `tests/fixtures/stacks` holds one project layout + per kind of stack, with the answer `detect` must give for it, and the suite checks each + one. A nightly job scaffolds Next.js, Astro, SvelteKit, Nuxt, Docusaurus and a Vite app + from their official starters, installs them and audits them with no directory. + +### Changed + +- **A single-page app's empty shell is no longer audited as a clean page.** A page with + nothing a visitor could perceive before a script runs, such as a Vite build's + `
`, is set aside and listed in `completeness.unreachable`. A build + that holds only a shell stops with exit 2 and the command to audit it with `--browser`, + which runs the script as before. +- **`storybook-static/` is never audited** as part of the site. +- **Servers are started on a free port,** offered through `PORT`. The address a server + prints is read through colour codes and `0.0.0.0`. The Angular, Gatsby, Hugo and Jekyll + default ports are also tried. + +### Fixed + +- **On Windows, a server the audit started is stopped with everything it started.** + Stopping only the `cmd.exe` that ran the script left the real server running after + the report was written, holding its port and its directory. + +### Report format + +- `completeness.discovery` can now be `"manifest"`: the pages came from the project's own + build, a Next.js build's manifests. `schemaVersion` stays 2, since no field was removed + or renamed. A consumer that switches on `discovery` should expect the new value. + ## 0.9.1 — 2026-09-26 ### Fixed diff --git a/README.md b/README.md index 4903b5d..1542de1 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,14 @@ the statute and supervisory body of **fifteen countries**: Austria, Belgium, Cze Denmark, Finland, France, Germany, Ireland, Italy, the Netherlands, Poland, Portugal, Spain, Sweden and Switzerland, each in its own language as well as English. +0.10.0 finds your site, whatever it is built with. A Next.js app that renders on a server +is built, started and audited page by page from its own build manifests, including pages +nothing links to. Forty-one stacks are recognised, from Qwik and TanStack Start to +MkDocs, Sphinx and Zola, and a monorepo's sites are found from its root. The package +manager comes from the project itself, Bun and Deno included. A single-page app's empty +shell is named as not audited instead of passing. `eaa-kit detect` says what the tool +makes of a project and why, and `eaa-kit doctor` checks everything it needs on one screen. + 0.9.0 needs no setup. `npx eaa-kit` on its own finds the site, audits it, writes an HTML report and says which command comes next. `init` fills itself in from what the built site states, and also writes a baseline and a GitHub Actions workflow tailored to the project. @@ -42,6 +50,8 @@ npx eaa-kit init # the config, a baseline and a CI workflow npx eaa-kit statement # accessibility statement, in one of fifteen countries npx eaa-kit countries # which ones, in which languages, under which law npx eaa-kit checklist # the manual review no engine can do for you +npx eaa-kit detect # what it makes of this project, and why +npx eaa-kit doctor # everything it needs here, checked on one screen ``` > **Not legal advice.** eaa-kit reports what an automated engine can and cannot determine @@ -71,8 +81,10 @@ answer. eaa-kit audit ./dist --fail-on serious ``` -Sites that render on a server and never write HTML to disk — Next.js without a static -export, Nuxt, SvelteKit, anything behind a CMS — are audited running instead: +Sites that render on a server and never write HTML to disk, such as Next.js without a +static export, Nuxt or SvelteKit, are built and started by `eaa-kit audit` with no +directory, and crawled while they run. Anything behind a CMS is never started uninvited; +start it yourself and audit it running: ```bash eaa-kit audit --url http://localhost:3000 diff --git a/ROADMAP.md b/ROADMAP.md index 18ea47a..4ba51d1 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -26,7 +26,9 @@ having to learn the tool first. These releases get there: the wrong version. - **0.10.0 — it finds your site.** Stack detection for what people actually build with (Next.js properly first), monorepos and package managers, and `eaa-kit detect` and - `eaa-kit doctor` to explain what it found. + `eaa-kit doctor` to explain what it found. *Done:* forty-one stacks, Next.js served and + crawled from its build manifests, single-page-app shells named instead of passed, a + recorded `detect` answer per stack fixture, and a nightly soak over six real starters. - **0.11.0 — it fits your workflow.** GitLab and Bitbucket CI from `init`, a Markdown summary for job summaries and PR comments, and an HTML report that prints, speaks the site's language and shows what changed since last time. diff --git a/docs/audit.md b/docs/audit.md index 8035b82..073fbeb 100644 --- a/docs/audit.md +++ b/docs/audit.md @@ -41,23 +41,45 @@ left untouched. Exit codes are those of `audit`. ## With no arguments -`eaa-kit audit` on its own works out what this project needs, in three steps: - -1. **A build that already exists.** The first of `dist/`, `out/`, `build/`, `_site/`, - `.output/public/` or `public/` that holds HTML. Holding HTML is the test, not merely - existing — `.next/` exists after any Next.js build and holds no browsable page, and - `public/` exists in most projects and holds assets. +`eaa-kit audit` on its own works out what this project needs, in three steps. +`eaa-kit detect` shows what it would decide, and why, without doing any of it. + +1. **A build that already exists.** The framework's own output directory first (`out/` + for a Next.js export, `_site/` for Eleventy, `site/` for MkDocs, whatever the config + file names), then `dist/`, `out/`, `build/`, `_site/`, `public/` or `.output/public/`, + whichever first holds HTML. Holding HTML is the test, not merely existing — `.next/` + exists after any Next.js build and holds no browsable page, and `public/` exists in + most projects and holds assets. `storybook-static/` is never counted: a component + catalogue is not the site. 2. **The project's own build.** Nothing built yet, so it runs the `build` script rather - than telling you to go and do it and come back. The package manager comes from the - lockfile. + than telling you to go and do it and come back. 3. **The project's server.** Built and still no HTML anywhere means the site renders on a - server — a Next.js app with an API route, middleware or ISR, and anything else that - cannot be exported. It starts `start`, `preview` or `serve`, crawls what that serves, - and stops it again afterwards. + server. It starts `start`, `preview` or `serve` on a free port, crawls what that serves, + and stops it again afterwards. For **Next.js** it reads the build's own manifests for + the list of pages, so a page nothing links to is still audited, and a dynamic route + with no prerendered pages is named as not audited rather than silently missed. It + respects `basePath` and i18n locales, serves a standalone build with its own + `server.js` when the static files are beside it, and never uses `next dev`. A folder with no `package.json` and HTML files at its top level is a site written by hand, and is audited where it stands. +**The package manager** is the one the project states: corepack's `packageManager` field +first, then the lockfile — `pnpm-lock.yaml`, `yarn.lock`, `bun.lock` or `bun.lockb`, +`deno.lock`, `package-lock.json` — looked for up to the repository root, so an app inside +a monorepo uses the workspace's. + +**In a monorepo** (pnpm, yarn or npm workspaces, Turborepo, Nx, Lerna), run from the root, +it looks for the packages that are sites. One site is audited as though the command had +been run inside it. Several are listed with the command for each, since auditing them +together would mix their pages into one report; `init` asks which one to set up. + +**A single-page app's shell** — an `index.html` holding an empty `
` and a +script — has nothing in it until the script runs, and the browserless engine does not run +scripts. Rather than audit the empty div and report the site clean, such a page is set +aside and [named as not audited](reports.md#completeness); a build that is only a shell +stops with the command to audit it in a browser, `--browser`, which runs the script. + Naming a directory or passing `--url` skips all of it, and `--no-build` stops it running anything, leaving step 1 only. @@ -65,6 +87,30 @@ Steps two and three run your project's own scripts. That is what a build-time to the Astro integration already audits from inside a build — but it is announced as it happens, and `--no-build` turns it off. +### `eaa-kit detect` + +``` +$ npx eaa-kit detect +Framework Next.js (package.json depends on next) +Package manager pnpm (found pnpm-lock.yaml) +Next.js pages 5 listed by the build (not listed, rendered on request: /user/[id]) + +An audit would run pnpm run build, start the site with pnpm run start, and crawl the pages its build lists. + → npx eaa-kit +``` + +Nothing is built, started or written. `--json` prints the same as data, which is also the +thing to paste into an issue when the answer is wrong. + +### `eaa-kit doctor` + +Everything this tool needs in a project, on one screen, each problem followed by the +command that fixes it: the Node.js version, whether the project's package manager is +installed, what detection makes of the site, the config and whether it parses, a GitHub, +GitLab or Bitbucket pipeline that runs eaa-kit, the baseline and any expired entries in +it, and Playwright with Chromium for `--browser`. It exits 2 only for what stops an audit +from running; a missing config, pipeline or baseline is advice. + ## Auditing a running site ```bash @@ -220,22 +266,28 @@ emit; it is not special. | Builder | Directory | Note | | --- | --- | --- | -| Astro, Vite, SvelteKit (static), Nuxt (generate) | `dist/`, `.output/public/` | ready as built | -| Eleventy, Hugo, Jekyll | `_site/`, `public/` | ready as built | -| Create React App | `build/` | one `index.html`; a client-rendered app has little in it | -| Next.js | `out/` | **only with `output: 'export'`** — see below | +| Astro, Vite, Vue CLI, Parcel, Rsbuild, Rspack, Ember, Qwik | `dist/` | ready as built; an app shell needs `--browser` | +| SvelteKit (static), Nuxt (generate), SolidStart, TanStack Start | `build/`, `.output/public/` | ready as built | +| Analog | `dist/analog/public/` | ready as built | +| Angular 17+ | `dist//browser/` | ready as built | +| Eleventy, Jekyll, Quarto | `_site/` | ready as built | +| Hugo, Hexo, Zola, Gatsby | `public/` | ready as built | +| Docusaurus, Create React App | `build/` | ready as built | +| VitePress | `.vitepress/dist/` | ready as built | +| MkDocs | `site/` | ready as built | +| Sphinx | `_build/html/` | ready as built | +| mdBook | `book/` | ready as built | +| Pelican | `output/` | ready as built | +| Next.js | `out/` | **only with `output: 'export'`**; otherwise it is served, see below | + +A directory set in the framework's config (`outDir`, `distDir`, `site_dir`, `output-dir`, +`build-dir`, `public_dir`, `OUTPUT_PATH`, `outputDir`) is read, never executed, and tried +first. **Next.js does not write HTML to `dist/`.** A default `next build` produces `.next/`, which -holds the server bundle rather than a browsable site. To audit the files, set -`output: 'export'` in `next.config.js`, run `next build`, and point eaa-kit at `out/`. - -That works only for a site with no server-side rendering, API routes, middleware or ISR. -If yours has any of those, do not fight the export — audit it running instead: - -```bash -npm run build && npx next start -eaa-kit audit --url http://localhost:3000 -``` +holds the server bundle rather than a browsable site. `eaa-kit audit` with no directory +handles that itself: it builds, starts `next start`, and crawls every page the build's +manifests list. With `output: 'export'`, the files in `out/` are audited instead. A run that reports `No HTML files found` means the directory exists but holds no `.html` — almost always the wrong directory rather than a clean site. @@ -780,7 +832,7 @@ a recorded result is a claim by a person, which is a different kind of thing. A CMS writes no browsable HTML to disk: every page is rendered per request, so there is no build directory to point at and never was one. `eaa-kit audit` recognises WordPress, TYPO3, -Craft, Laravel, Symfony, Rails and Django, and rather than reporting an empty `./dist` it +Drupal, Craft, Statamic, Laravel, Symfony, Rails, Django, and Ghost and Shopify themes, and rather than reporting an empty `./dist` it says what the project is and how to audit it: ``` diff --git a/docs/integrations.md b/docs/integrations.md index 2da4c48..b6ce3ee 100644 --- a/docs/integrations.md +++ b/docs/integrations.md @@ -250,7 +250,7 @@ jobs: - uses: actions/setup-node@v4 with: node-version: 22 - - uses: likeBloodMoon/eaa-kit@v0.9.1 + - uses: likeBloodMoon/eaa-kit@v0.10.0 with: install-command: npm ci build-command: npm run build diff --git a/docs/reports.md b/docs/reports.md index d27f8b7..5244a59 100644 --- a/docs/reports.md +++ b/docs/reports.md @@ -88,7 +88,7 @@ A complete generated document is checked in at // are. Read "complete" before drawing any conclusion from the counts above. "completeness": { "complete": true, // false when anything went unmeasured - "discovery": "directory", // "directory" | "sitemap" | "links" + "discovery": "directory", // "directory" | "manifest" | "sitemap" | "links" "collected": 5, // pages handed to the engine "audited": 5, // pages this run reached a verdict on "reused": 0, // pages whose result came from the cache diff --git a/examples/report.html b/examples/report.html index 70db090..e9d7713 100644 --- a/examples/report.html +++ b/examples/report.html @@ -3,7 +3,7 @@ - + Accessibility audit · tests/fixtures/site