From 9e5694fd82c9a0739d0e3f545e3cdc16cb1b5ff3 Mon Sep 17 00:00:00 2001 From: Noel Tock Date: Tue, 15 Sep 2026 07:23:27 +0700 Subject: [PATCH 1/2] ci: route documentation and demo pushes conservatively --- .github/workflows/ci.yml | 63 +++++++++++++++++++++++++++++++------- dev/test/ci-scope.test.mjs | 22 +++++++++---- scripts/ci-scope.mjs | 9 ++++-- 3 files changed, 75 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 74e0cf7..f32edc8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,6 +15,8 @@ jobs: outputs: route: ${{ steps.scope.outputs.route }} docs: ${{ steps.scope.outputs.docs }} + base: ${{ steps.scope.outputs.base }} + head: ${{ steps.scope.outputs.head }} steps: - uses: actions/checkout@v4 with: @@ -24,18 +26,36 @@ jobs: - id: scope name: Classify changed paths shell: bash + env: + EVENT_NAME: ${{ github.event_name }} + PR_BASE: ${{ github.event.pull_request.base.sha }} + PR_HEAD: ${{ github.event.pull_request.head.sha }} + PUSH_BEFORE: ${{ github.event.before }} + PUSH_AFTER: ${{ github.event.after }} run: | - if [ "${{ github.event_name }}" != "pull_request" ]; then - echo 'route=full' >> "$GITHUB_OUTPUT" - echo 'docs=[]' >> "$GITHUB_OUTPUT" - exit 0 + set -euo pipefail + if [ "$EVENT_NAME" = pull_request ]; then + base="$PR_BASE" + head="$PR_HEAD" + elif [ "$EVENT_NAME" = push ]; then + base="$PUSH_BEFORE" + head="$PUSH_AFTER" + else + echo "Unsupported event for CI scope classification: $EVENT_NAME" >&2 + exit 1 + fi + if [ -z "$base" ] || [ -z "$head" ] || [[ "$base" =~ ^0+$ ]] || [[ "$head" =~ ^0+$ ]]; then + result='{"route":"full","docs":[],"reason":"missing-range-boundary"}' + else + result="$(node scripts/ci-scope.mjs --base "$base" --head "$head")" fi - result="$(node scripts/ci-scope.mjs --base '${{ github.event.pull_request.base.sha }}' --head '${{ github.event.pull_request.head.sha }}')" echo "$result" route="$(node --input-type=module --eval "console.log(JSON.parse(process.argv[1]).route)" "$result")" docs="$(node --input-type=module --eval "console.log(JSON.stringify(JSON.parse(process.argv[1]).docs))" "$result")" echo "route=$route" >> "$GITHUB_OUTPUT" echo "docs=$docs" >> "$GITHUB_OUTPUT" + echo "base=$base" >> "$GITHUB_OUTPUT" + echo "head=$head" >> "$GITHUB_OUTPUT" documentation-contracts: needs: classify @@ -44,8 +64,8 @@ jobs: steps: - uses: actions/checkout@v4 with: - # The local-link gate diffs the pull-request base and head. A rebased - # head does not retain that base in checkout's default shallow clone. + # The local-link gate diffs the event's base and head. A rebased PR + # head does not retain its base in checkout's default shallow clone. fetch-depth: 0 - uses: actions/setup-node@v4 with: @@ -59,8 +79,10 @@ jobs: - name: Check touched documentation whitespace and local links env: DOCS: ${{ needs.classify.outputs.docs }} + BASE: ${{ needs.classify.outputs.base }} + HEAD: ${{ needs.classify.outputs.head }} run: | - git diff --check '${{ github.event.pull_request.base.sha }}' '${{ github.event.pull_request.head.sha }}' + git diff --check "$BASE" "$HEAD" node --input-type=module <<'NODE' import { existsSync, readFileSync } from 'node:fs'; import path from 'node:path'; @@ -79,6 +101,23 @@ jobs: } } NODE + - name: Verify demo contracts + run: | + set -euo pipefail + bash -n demo/demo.sh + test -s demo/demo.gif + test -s assets/benchmark.jpg + node --input-type=module <<'NODE' + import { readFileSync } from 'node:fs'; + + const tape = readFileSync('demo/demo.tape', 'utf8'); + if (!tape.includes('Type `clear; bash demo/demo.sh`')) { + throw new Error('demo/demo.tape must run demo/demo.sh'); + } + if (!tape.includes('Output demo/demo.hq.gif') || !tape.includes('Output demo/demo.mp4')) { + throw new Error('demo/demo.tape must retain its documented output formats'); + } + NODE skill-contracts: needs: classify @@ -87,8 +126,8 @@ jobs: steps: - uses: actions/checkout@v4 with: - # The local-link gate diffs the pull-request base and head. A rebased - # head does not retain that base in checkout's default shallow clone. + # The local-link gate diffs the event's base and head. A rebased PR + # head does not retain its base in checkout's default shallow clone. fetch-depth: 0 - uses: actions/setup-node@v4 with: @@ -102,8 +141,10 @@ jobs: - name: Check touched documentation whitespace and local links env: DOCS: ${{ needs.classify.outputs.docs }} + BASE: ${{ needs.classify.outputs.base }} + HEAD: ${{ needs.classify.outputs.head }} run: | - git diff --check '${{ github.event.pull_request.base.sha }}' '${{ github.event.pull_request.head.sha }}' + git diff --check "$BASE" "$HEAD" node --input-type=module <<'NODE' import { existsSync, readFileSync } from 'node:fs'; import path from 'node:path'; diff --git a/dev/test/ci-scope.test.mjs b/dev/test/ci-scope.test.mjs index 5b90712..8675111 100644 --- a/dev/test/ci-scope.test.mjs +++ b/dev/test/ci-scope.test.mjs @@ -4,10 +4,14 @@ import { classifyChanges, classifyDiffResult, parseNameStatus } from '../../scri const entry = (status, ...paths) => ({ status, paths }); -test('selects the documentation route only for the root documentation allowlist', () => { +test('selects the documentation route only for the explicit documentation and demo allowlists', () => { assert.deepEqual( classifyChanges([entry('M', 'README.md'), entry('A', 'CHANGELOG.md')]), - { route: 'docs', docs: ['CHANGELOG.md', 'README.md'], reason: 'root-documentation-only' }, + { route: 'docs', docs: ['CHANGELOG.md', 'README.md'], reason: 'documentation-and-demo-only' }, + ); + assert.deepEqual( + classifyChanges([entry('M', 'README.md'), entry('M', 'demo/demo.sh'), entry('M', 'demo/demo.gif')]), + { route: 'docs', docs: ['README.md'], reason: 'documentation-and-demo-only' }, ); }); @@ -34,14 +38,20 @@ test('keeps unknown Markdown on the full route', () => { assert.equal(classifyChanges([entry('M', 'dev/benchmarks/README.md')]).route, 'full'); }); -test('uses both sides of a rename', () => { +test('keeps renames and copies on the full route even when every path is allowlisted', () => { assert.equal(classifyChanges([entry('R100', 'README.md', 'src/README.md')]).route, 'full'); - assert.equal(classifyChanges([entry('R100', 'skills/block-runner/SKILL.md', 'README.md')]).route, 'skill'); + assert.equal(classifyChanges([entry('R100', 'skills/block-runner/SKILL.md', 'README.md')]).route, 'full'); + assert.equal(classifyChanges([entry('C100', 'README.md', 'CHANGELOG.md')]).route, 'full'); }); -test('uses deleted paths', () => { +test('keeps deletions on the full route even when the deleted path is allowlisted', () => { assert.equal(classifyChanges([entry('D', 'scripts/old-check.mjs')]).route, 'full'); - assert.equal(classifyChanges([entry('D', 'ERRORS.md')]).route, 'docs'); + assert.equal(classifyChanges([entry('D', 'ERRORS.md')]).route, 'full'); +}); + +test('keeps type changes and unmerged paths on the full route', () => { + assert.equal(classifyChanges([entry('T', 'README.md')]).route, 'full'); + assert.equal(classifyChanges([entry('U', 'README.md')]).route, 'full'); }); test('keeps empty input and unavailable or malformed diffs on the full route', () => { diff --git a/scripts/ci-scope.mjs b/scripts/ci-scope.mjs index d3dbdc5..8bb33e6 100644 --- a/scripts/ci-scope.mjs +++ b/scripts/ci-scope.mjs @@ -3,6 +3,7 @@ import { spawnSync } from 'node:child_process'; const DOCUMENTATION_PATHS = new Set(['README.md', 'CHANGELOG.md', 'ERRORS.md', 'DECISIONS.md']); +const DEMO_PATHS = new Set(['demo/demo.sh', 'demo/demo.tape', 'demo/demo.gif', 'assets/benchmark.jpg']); const SKILL_PREFIX = 'skills/block-runner/'; /** @@ -15,6 +16,10 @@ export function classifyChanges(entries) { return { route: 'full', docs: [], reason: 'empty-diff' }; } + if (entries.some((entry) => !entry || (entry.status !== 'A' && entry.status !== 'M'))) { + return { route: 'full', docs: [], reason: 'non-additive-or-unknown-change' }; + } + const paths = entries.flatMap((entry) => entry.paths ?? []); if (paths.length === 0 || paths.some((path) => typeof path !== 'string' || path.length === 0)) { return { route: 'full', docs: [], reason: 'unclassified-path' }; @@ -22,8 +27,8 @@ export function classifyChanges(entries) { const docs = [...new Set(paths.filter((path) => DOCUMENTATION_PATHS.has(path) || (path.startsWith(SKILL_PREFIX) && path.endsWith('.md'))))].sort(); - if (paths.every((path) => DOCUMENTATION_PATHS.has(path))) { - return { route: 'docs', docs, reason: 'root-documentation-only' }; + if (paths.every((path) => DOCUMENTATION_PATHS.has(path) || DEMO_PATHS.has(path))) { + return { route: 'docs', docs, reason: 'documentation-and-demo-only' }; } if (paths.every((path) => DOCUMENTATION_PATHS.has(path) || path.startsWith(SKILL_PREFIX)) From 57f15167ea5ec762dd1e5e5660a8334557d8aca7 Mon Sep 17 00:00:00 2001 From: Noel Tock Date: Tue, 15 Sep 2026 07:23:27 +0700 Subject: [PATCH 2/2] test: reuse WordPress across sequential proof cases --- dev/test/proof-real-wordpress.test.ts | 21 +++++++- dev/test/proof-wordpress-lifecycle.test.ts | 56 ++++++++++++++++++++++ dev/test/proof-wordpress-lifecycle.ts | 42 ++++++++++++++++ 3 files changed, 117 insertions(+), 2 deletions(-) create mode 100644 dev/test/proof-wordpress-lifecycle.test.ts create mode 100644 dev/test/proof-wordpress-lifecycle.ts diff --git a/dev/test/proof-real-wordpress.test.ts b/dev/test/proof-real-wordpress.test.ts index 5d8f0bb..655c8c3 100644 --- a/dev/test/proof-real-wordpress.test.ts +++ b/dev/test/proof-real-wordpress.test.ts @@ -4,8 +4,9 @@ import { mkdir, mkdtemp, readFile, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import path from 'node:path'; import { promisify } from 'node:util'; -import { describe, expect, it } from 'vitest'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { buildNativeStyleAdapterProofFixture, buildPatternOverridesFixture, buildResponsiveStyleProofFixture } from '../../scripts/build-pattern-overrides-fixture.js'; +import { createWordPressProofLifecycle } from './proof-wordpress-lifecycle.js'; import { PROOF_PROFILES, canonicalJson, @@ -22,6 +23,7 @@ import { } from '../../src/proof/release-acceptance.js'; const execFileAsync = promisify(execFile); +const wpEnvConfig = path.resolve('proof/wp-env.json'); type ResponsiveStyleMatrixEvidence = { scope?: string; @@ -53,6 +55,16 @@ type NativeStyleAdapterMatrixEvidence = { describe('real WordPress generated-pattern full-profile receipt', () => { let scopedFixture: Awaited>; let historicalGates: Awaited>['receipt']['gates']; + const lifecycle = createWordPressProofLifecycle(async (args) => { + const { stdout, stderr } = await execFileAsync('npx', ['--no-install', 'wp-env', `--config=${wpEnvConfig}`, ...args], { + timeout: args[0] === 'stop' ? 60_000 : 45_000, + }); + return { stdout, stderr }; + }); + + beforeAll(() => lifecycle.prepare(), 75_000); + afterAll(() => lifecycle.cleanup(), 75_000); + it('writes a complete raw WordPress 7.1 receipt with the retained root-grid iframe matrix and a separate acceptance assessment', async () => { await requireDocker(); const outputDir = await proofOutputDirectory(); @@ -73,6 +85,7 @@ describe('real WordPress generated-pattern full-profile receipt', () => { fixture: built.fixture, artifact: built.artifact, outputDir, + keepEnvironment: true, }); historicalGates = result.receipt.gates; @@ -215,6 +228,7 @@ describe('real WordPress generated-pattern full-profile receipt', () => { fixture: built.fixture, artifact: built.artifact, outputDir, + keepEnvironment: true, }); const editor = result.receipt.gates.find((gate) => gate.gate === 'editor_reopen'); const frontend = result.receipt.gates.find((gate) => gate.gate === 'frontend_assets'); @@ -254,6 +268,7 @@ describe('real WordPress generated-pattern full-profile receipt', () => { fixture: built.fixture, artifact: built.artifact, outputDir, + keepEnvironment: true, }); const editor = result.receipt.gates.find((gate) => gate.gate === 'editor_reopen'); const frontend = result.receipt.gates.find((gate) => gate.gate === 'frontend_assets'); @@ -297,7 +312,7 @@ describe('real WordPress generated-pattern full-profile receipt', () => { const built = scopedFixture; const outputDir = await proofOutputDirectory(profile); const result = await runProof({ profile, pluginZip: built.pluginZip, artifact: built.artifact, - inputPath: built.inputPath, markup: built.nativeContainerMarkup, fixture: built.fixture, outputDir }); + inputPath: built.inputPath, markup: built.nativeContainerMarkup, fixture: built.fixture, outputDir, keepEnvironment: true }); expect(result.receipt.requirements?.missingInputs).toEqual([]); // Preserve upstream findings, not an artificial all-pass expectation. Every requested gate // must execute just as it did for the same artifact under the historical full profile. @@ -323,6 +338,7 @@ describe('real WordPress generated-pattern full-profile receipt', () => { fixture: built.fixture, artifact: built.artifact, outputDir, + keepEnvironment: true, }); const matrix = (result.receipt.gates.find((gate) => gate.gate === 'editor_reopen')?.details as { browserMatrix?: { @@ -349,6 +365,7 @@ describe('real WordPress generated-pattern full-profile receipt', () => { const result = await runProof({ profile: 'editor', pluginZip: built.pluginZip, inputPath: built.inputPath, markup: built.nativeContainerMarkup, fixture: built.fixture, outputDir, + keepEnvironment: true, }); const matrix = (result.receipt.gates.find((gate) => gate.gate === 'editor_reopen')?.details as { browserMatrix?: { iframe?: { observed?: boolean }; rootLayout?: string; beforeAfter?: { ok?: boolean; directNativeChildren?: string[] } }; diff --git a/dev/test/proof-wordpress-lifecycle.test.ts b/dev/test/proof-wordpress-lifecycle.test.ts new file mode 100644 index 0000000..91db4a5 --- /dev/null +++ b/dev/test/proof-wordpress-lifecycle.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from 'vitest'; +import { createWordPressProofLifecycle } from './proof-wordpress-lifecycle.js'; + +describe('WordPress proof lifecycle', () => { + it('stops an environment this suite found stopped even when a proof fails', async () => { + const commands: string[][] = []; + const lifecycle = createWordPressProofLifecycle(async (args) => { + commands.push([...args]); + return { stdout: args[0] === 'status' ? '{"status":"uninitialized"}' : '', stderr: '' }; + }); + + await lifecycle.prepare(); + await lifecycle.cleanup(); + + expect(commands).toEqual([['status', '--json'], ['stop']]); + }); + + it('leaves a pre-existing environment for its owner', async () => { + const commands: string[][] = []; + const lifecycle = createWordPressProofLifecycle(async (args) => { + commands.push([...args]); + return { stdout: '{"status":"running"}', stderr: '' }; + }); + + await lifecycle.prepare(); + await lifecycle.cleanup(); + + expect(commands).toEqual([['status', '--json']]); + }); + + it('refuses an unknown status without stopping an environment', async () => { + const commands: string[][] = []; + const lifecycle = createWordPressProofLifecycle(async (args) => { + commands.push([...args]); + return { stdout: '{"status":"unknown"}', stderr: '' }; + }); + + await expect(lifecycle.prepare()).rejects.toThrow('Unknown wp-env runtime status'); + await lifecycle.cleanup(); + + expect(commands).toEqual([['status', '--json']]); + }); + + it('refuses a failed status probe without stopping an environment', async () => { + const commands: string[][] = []; + const lifecycle = createWordPressProofLifecycle(async (args) => { + commands.push([...args]); + throw new Error('wp-env status failed'); + }); + + await expect(lifecycle.prepare()).rejects.toThrow('wp-env status failed'); + await lifecycle.cleanup(); + + expect(commands).toEqual([['status', '--json']]); + }); +}); diff --git a/dev/test/proof-wordpress-lifecycle.ts b/dev/test/proof-wordpress-lifecycle.ts new file mode 100644 index 0000000..9329ec8 --- /dev/null +++ b/dev/test/proof-wordpress-lifecycle.ts @@ -0,0 +1,42 @@ +export type WordPressEnvironmentStatus = { + stdout: string; + stderr: string; +}; + +export type WordPressEnvironmentCommand = (args: readonly string[]) => Promise; + +/** + * Keep a sequential proof suite on one wp-env lifecycle without stopping an + * environment that was already running before the suite began. + */ +export function createWordPressProofLifecycle(command: WordPressEnvironmentCommand): { + prepare(): Promise; + cleanup(): Promise; +} { + let startedHere = false; + + return { + async prepare() { + const status = await command(['status', '--json']); + let parsed: unknown; + try { + parsed = JSON.parse(status.stdout); + } catch { + throw new Error('wp-env status did not return JSON; refusing to claim lifecycle ownership.'); + } + if (!parsed || typeof parsed !== 'object' || !('status' in parsed)) { + throw new Error('wp-env status did not return a runtime status; refusing to claim lifecycle ownership.'); + } + const runtimeStatus = parsed.status; + if (runtimeStatus === 'running') return; + if (runtimeStatus === 'uninitialized' || runtimeStatus === 'stopped') { + startedHere = true; + return; + } + throw new Error(`Unknown wp-env runtime status: ${String(runtimeStatus)}; refusing to claim lifecycle ownership.`); + }, + async cleanup() { + if (startedHere) await command(['stop']); + }, + }; +}