test: version-pinned end-to-end suite for generated apps

.github/workflows/e2e.yaml

@@ -0,0 +1,119 @@
+name: E2E
+on:
+  # Run on demand against any Harper version (a dist-tag like `latest`/`next`, an explicit
+  # version like `5.2.0-beta.1`, or any npm spec — including a tarball or git url).
+  workflow_dispatch:
+    inputs:
+      harper-version:
+        description: 'Harper npm spec to test (latest, next, 5.2.0-beta.1, a tarball/git url, ...)'
+        default: 'latest'
+        required: true
+      harper-ref:
+        description: 'Build Harper from this git commit/branch/tag instead (overrides harper-version)'
+        default: ''
+        required: false
+      template:
+        description: 'Template to test, or "all" for the representative matrix'
+        default: 'all'
+        required: true
+  # Nightly canary against the leading edge — the early warning for Harper prereleases breaking
+  # a generated app. Opens an issue on failure (see the notify job).
+  schedule:
+    - cron: '0 7 * * *'
+
+concurrency:
+  group: e2e-${{ github.ref }}-${{ github.event_name }}
+  cancel-in-progress: true
+
+jobs:
+  matrix:
+    name: Compute matrix
+    runs-on: ubuntu-latest
+    outputs:
+      templates: ${{ steps.set.outputs.templates }}
+      versions: ${{ steps.set.outputs.versions }}
+    steps:
+      - id: set
+        shell: bash
+        run: |
+          # Templates: a manually chosen single template, otherwise a representative subset —
+          # one per frontend family plus one SSR variant. The package manager is irrelevant to
+          # Harper's API surface, so (unlike the PR integration test) this axis is npm-only.
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "${{ inputs.template }}" != "all" ]; then
+            echo "templates=[\"${{ inputs.template }}\"]" >> "$GITHUB_OUTPUT"
+          else
+            echo 'templates=["vanilla-ts","react-ts","vue-ts","react-ts-ssr"]' >> "$GITHUB_OUTPUT"
+          fi
+          # Versions: a single cell when building from a git ref (the ref is the only dimension),
+          # the dispatched npm spec, otherwise latest + next for the nightly canary.
+          if [ -n "${{ inputs.harper-ref }}" ]; then
+            echo 'versions=["(source)"]' >> "$GITHUB_OUTPUT"
+          elif [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            echo 'versions=["${{ inputs.harper-version }}"]' >> "$GITHUB_OUTPUT"
+          else
+            echo 'versions=["latest","next"]' >> "$GITHUB_OUTPUT"
+          fi
+
+  e2e:
+    needs: matrix
+    name: ${{ matrix.template }} @ harper@${{ matrix.harper-version }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        template: ${{ fromJSON(needs.matrix.outputs.templates) }}
+        harper-version: ${{ fromJSON(needs.matrix.outputs.versions) }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+
+      - name: Setup Node.js
+        uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0
+        with:
+          node-version-file: '.nvmrc'
+          cache: npm
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Cache Playwright browsers
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ~/.cache/ms-playwright
+          key: ${{ runner.os }}-playwright-${{ hashFiles('package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-playwright-
+
+      - name: Run E2E (${{ matrix.template }} against harper ${{ matrix.harper-version }})
+        run: |
+          if [ -n "${{ inputs.harper-ref }}" ]; then
+            node template.tests/e2e/run.js --template ${{ matrix.template }} --harper-ref "${{ inputs.harper-ref }}"
+          else
+            node template.tests/e2e/run.js --template ${{ matrix.template }} --harper "${{ matrix.harper-version }}"
+          fi
+
+  notify:
+    needs: e2e
+    # Only the scheduled canary auto-files an issue; manual runs surface in the Actions UI.
+    if: ${{ failure() && github.event_name == 'schedule' }}
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+    env:
+      GH_TOKEN: ${{ github.token }}
+      GH_REPO: ${{ github.repository }}
+    steps:
+      - name: File an issue for the failing canary (deduped)
+        run: |
+          gh label create e2e-canary --description "Nightly E2E canary failures" --color B60205 --force || true
+          open=$(gh issue list --label e2e-canary --state open --json number --jq 'length')
+          run_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          if [ "$open" = "0" ]; then
+            gh issue create --label e2e-canary \
+              --title "E2E canary failing against a Harper prerelease" \
+              --body "The nightly E2E canary failed — a generated app may break on an upcoming Harper release. Run: ${run_url}"
+          else
+            echo "An open e2e-canary issue already exists; commenting instead of duplicating."
+            number=$(gh issue list --label e2e-canary --state open --json number --jq '.[0].number')
+            gh issue comment "$number" --body "Canary failed again: ${run_url}"
+          fi

.gitignore

@@ -1,6 +1,12 @@
 .DS_Store
 .temp-integration-tests
 
+# Playwright e2e artifacts
+/test-results/
+/playwright-report/
+/blob-report/
+/playwright/.cache/
+
 #
 # https://raw.githubusercontent.com/github/gitignore/refs/heads/main/Node.gitignore
 #

.oxlintrc.json

@@ -1,6 +1,6 @@
 {
 	"$schema": "./node_modules/oxlint/configuration_schema.json",
-	"ignorePatterns": [],
+	"ignorePatterns": ["template.tests/e2e/fixtures/**"],
 	"rules": {},
 	"overrides": [
 		{

dprint.json

@@ -18,7 +18,8 @@
 	},
 	"excludes": [
 		"**/node_modules",
-		"template-early-hints/schemas/schema.graphql"
+		"template-early-hints/schemas/schema.graphql",
+		"template.tests/e2e/fixtures"
 	],
 	"plugins": [
 		"https://plugins.dprint.dev/typescript-0.95.13.wasm",

package.json

@@ -43,6 +43,7 @@
 		"test": "vitest run",
 		"test:coverage": "vitest run --coverage",
 		"test:watch": "vitest",
+		"test:e2e": "node template.tests/e2e/run.js",
 		"templates:apply-shared-templates": "(cd templates-shared && node ./applySharedTemplates.js)",
 		"templates:build-studio-templates": "(cd templates-studio && node ./buildStudioTemplates.js)",
 		"templates:publish-studio-templates": "(cd templates-studio && node ./publishStudioTemplates.js)"
@@ -65,6 +66,7 @@
 	"devDependencies": {
 		"@commitlint/cli": "^21.0.0",
 		"@commitlint/config-conventional": "^21.0.0",
+		"@playwright/test": "^1.61.1",
 		"@semantic-release/commit-analyzer": "^13.0.1",
 		"@semantic-release/git": "^10.0.1",
 		"@semantic-release/github": "^12.0.2",

template.tests/e2e/README.md

@@ -0,0 +1,112 @@
+# End-to-end tests
+
+These tests scaffold a real app with the create-harper CLI, exercise it against a **real Harper
+instance**, and let you pin **which version of Harper** to run against — so we can catch, ahead
+of a Harper release, anything that would break a generated app: changed API surfaces, new
+instability, or defaults create-harper needs to follow.
+
+This is the deeper tier above [`../runtimeSmoke.js`](../runtimeSmoke.js) (the fast per-PR check).
+It shares the same isolated-Harper boot module, [`harper.js`](./harper.js).
+
+## Running locally
+
+```bash
+# One template against the latest published Harper:
+npm run test:e2e -- --template react-ts
+
+# Against the leading edge (the `next` dist-tag), or any npm spec:
+npm run test:e2e -- --template react-ts --harper next
+npm run test:e2e -- --template vue-ts   --harper 5.2.0-beta.1
+
+# Against an UNPUBLISHED Harper, built from a git branch / tag / full commit sha:
+npm run test:e2e -- --template react-ts --harper-ref my-feature-branch
+npm run test:e2e -- --template react-ts --harper-ref 1e1edc666ad373a0fbfec4df4d3f0e130be13529
+npm run test:e2e -- --template react-ts --harper-ref my-branch --harper-repo https://github.com/me/harper.git
+
+# Reuse a Harper you already have installed/built (skips resolution entirely):
+HARPER_BIN=/path/to/node_modules/.bin/harper npm run test:e2e -- --template vanilla-ts
+
+# Keep the generated app around to inspect a failure:
+npm run test:e2e -- --template react-ts --keep
+```
+
+How Harper is resolved, in precedence order:
+
+- `HARPER_BIN` (env) — an existing install or build; `run.js` uses it as-is. A `.js` path (e.g. a
+  source build's `dist/bin/harper.js`) is launched via `node`.
+- `--harper-ref <sha|branch|tag>` — clone `harperfast/harper` (public; override with
+  `--harper-repo`), `npm install` + `npm run build`, and run its `dist/bin/harper.js`. This tests
+  an unpublished Harper straight from a commit. Heaviest path (full install + `tsc` build). A commit
+  ref must be a **full 40-char sha** (git can't fetch a short sha directly); branches and tags work
+  as-is. The `build` script is `tsc` and Harper's `main` isn't always type-green, so — like Harper's
+  own `build-tools/build.sh` (`npm run build || true`) — a non-zero build exit is tolerated as long
+  as `dist/bin/harper.js` was emitted.
+- `--harper <spec>` (default `latest`) — `npm install harper@<spec>` into an isolated prefix. Any
+  npm spec: `latest`, `next`, `5.2.0-beta.1`, a tarball, ...
+
+Nothing is installed globally in any case.
+
+## What a run does
+
+[`run.js`](./run.js) drives one template end-to-end:
+
+1. **Generate** the app with the real CLI (throwaway temp dir).
+2. **Overlay** e2e fixtures ([`overlay.js`](./overlay.js)) — see "no cruft" below.
+3. **Install** the app's dependencies.
+4. **Resolve Harper** (no global install): a published version via npm, or an unpublished build
+   from a git ref — see the precedence list above.
+5. **Build** the app if it has a build step.
+6. **Playwright** ([`playwright.config.js`](./playwright.config.js)) boots one isolated Harper via
+   [`globalSetup.js`](./globalSetup.js), seeds a known record, and runs the specs against it.
+
+## Scenarios
+
+Every template runs the API specs (CRUD + custom resource — the framework-agnostic Harper
+surface). SPA templates additionally run the frontend-served and browser specs.
+
+| Spec                                           | What it proves                                                                                                                      | Templates                   |
+| ---------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- | --------------------------- |
+| [`crud.spec.js`](./specs/crud.spec.js)         | Schema-driven auto-REST CRUD lifecycle (create/read/update/filter/delete), with the exact status codes asserted as a version canary | all                         |
+| [`resource.spec.js`](./specs/resource.spec.js) | A custom `resources/` class loads and answers GET with its own JSON                                                                 | all                         |
+| [`frontend.spec.js`](./specs/frontend.spec.js) | The frontend is served at `/`                                                                                                       | react / vue / vanilla (SPA) |
+| [`browser.spec.js`](./specs/browser.spec.js)   | A real framework component fetches the auto-REST API and renders the data in a browser                                              | react / vue / vanilla (SPA) |
+
+> Note: there is **no GraphQL query-over-HTTP endpoint** in Harper (verified: `/graphql` → 404).
+> "GraphQL" in the templates means the schema definition language plus the auto-generated REST +
+> WebSocket APIs — which is what these specs exercise.
+
+## No cruft in real users' apps
+
+The fixtures under [`fixtures/`](./fixtures) are **only** ever copied into throwaway generated
+copies, never into the `template-*/` sources, and `template.tests/` is excluded from the npm
+`files` allowlist. So none of this can reach a user's scaffolded app or the published package.
+
+- **Shared** ([`fixtures/shared`](./fixtures/shared)) — a schema (`E2eWidget`) + a custom resource
+  (`E2eEcho`), applied to every template. This is Harper's API surface, which is framework-agnostic.
+- **Framework** ([`fixtures/react`](./fixtures/react), [`fixtures/vue`](./fixtures/vue)) — a
+  component that fetches `E2eWidget` and renders it, mounted into the app's entry by
+  [`overlay.js`](./overlay.js). Vanilla gets an equivalent plain-DOM overlay.
+
+## CI
+
+- **Per PR** — the fast `runtimeSmoke.js` check (via `.github/workflows/integration.yaml`), against
+  the latest published Harper.
+- **Nightly + on demand** — `.github/workflows/e2e.yaml` runs this suite. The scheduled run tests
+  `latest` **and** `next` across a representative template subset and files an issue if the `next`
+  canary breaks. `workflow_dispatch` takes `harper-version` (an npm spec), `harper-ref` (a git
+  commit/branch/tag to build from source — overrides `harper-version`), and optional `template`.
+
+## Extending
+
+- **SSR frontend + browser coverage**: SSR templates (`*-ssr`) currently run the API specs only.
+  Two things differ from SPA: their entry is an `entry-client`/`entry-server` split rather than a
+  single `main` (so the component overlay is gated off — `capabilities.js` → `browser: false`), and
+  their `static` handler sets `index: false` so `/` is served by the SSR entry-server. In local
+  validation that path returned **404 under `harper run`** — so the frontend spec is gated off for
+  SSR too (`E2E_SSR`). Confirming the intended production SSR serving, then enabling both specs for
+  SSR, is the natural next step (and a good first real use of this harness).
+- **More scenarios**: add a `*.spec.js` under [`specs/`](./specs). API specs use the `request`
+  fixture (authenticated as the seeded superuser); browser specs use `page` and should be gated on
+  `E2E_BROWSER`.
+- **New capabilities per template**: [`capabilities.js`](./capabilities.js) derives everything from
+  the template name.

template.tests/e2e/capabilities.js

@@ -0,0 +1,22 @@
+/**
+ * Derives what an e2e run should do for a given template, entirely from its canonical name.
+ *
+ *   - ext:       resources are `.ts` for TypeScript templates, `.js` otherwise.
+ *   - framework: which frontend overlay to apply (react / vue / vanilla).
+ *   - ssr:       server-rendered templates use an entry-client/entry-server split rather than a
+ *                single `main` entry, so the browser-component overlay doesn't apply (yet). They
+ *                still run the full HTTP suite (CRUD, custom resource, frontend-served).
+ *   - browser:   whether to apply the frontend component overlay and run the browser spec.
+ */
+
+/** @param {string} templateName */
+export function capabilitiesFor(templateName) {
+	const ext = templateName.includes('-ts') ? 'ts' : 'js';
+	const framework = templateName.startsWith('react')
+		? 'react'
+		: templateName.startsWith('vue')
+		? 'vue'
+		: 'vanilla';
+	const ssr = templateName.endsWith('-ssr');
+	return { templateName, ext, framework, ssr, browser: !ssr };
+}