From 44f10e33604c2abfb659baca965006a0c2e545f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Val=20Alexander=20=F0=9F=91=91?= Date: Mon, 14 Sep 2026 01:35:27 -0500 Subject: [PATCH] fix(ci): preserve browser evidence and pin source freshness Run documentation browser certification independently of source freshness, while preserving the canonical fail-closed release rollup. Compare watched paths in pinned Git trees rather than timestamp-filtered moving refs. Add 83 dependency-free CLI and workflow regressions. Refs #89; this does not advance the source lock or close the outstanding public-source review. --- .github/workflows/docs.yml | 70 ++++++-- docs/e2e-source-freshness.md | 74 ++++++++ scripts/check-automation-syntax.mjs | 10 ++ scripts/check-source-drift.mjs | 216 ++++++++++++++++------- scripts/check-source-drift.test.mjs | 255 ++++++++++++++++++++++++++++ scripts/docs-release-gate.mjs | 17 ++ scripts/docs-release-gate.test.mjs | 41 +++++ 7 files changed, 609 insertions(+), 74 deletions(-) create mode 100644 docs/e2e-source-freshness.md create mode 100644 scripts/check-source-drift.test.mjs create mode 100644 scripts/docs-release-gate.mjs create mode 100644 scripts/docs-release-gate.test.mjs diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 79b3b63..f5a7ce3 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -15,8 +15,42 @@ concurrency: cancel-in-progress: true jobs: - verify: - name: Verify documentation release + freshness: + name: Verify upstream documentation source + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Install Node.js + uses: actions/setup-node@v4 + with: + node-version: 22 + + - name: Test freshness and release-gate invariants + run: node --test scripts/check-source-drift.test.mjs scripts/docs-release-gate.test.mjs + + - name: Validate source lock + run: node scripts/check-source-lock.mjs + + - name: Verify upstream source freshness + env: + GITHUB_TOKEN: ${{ github.token }} + DOCS_DRIFT_REPORT_PATH: output/docs-source-drift.json + run: node scripts/check-source-drift.mjs + + - name: Upload source-drift evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: docs-source-drift-${{ github.run_id }} + path: output/docs-source-drift.json + if-no-files-found: warn + retention-days: 30 + + browser: + name: Certify documentation build and browser behavior runs-on: ubuntu-latest timeout-minutes: 30 env: @@ -43,12 +77,6 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile - - name: Verify upstream source freshness - env: - GITHUB_TOKEN: ${{ github.token }} - DOCS_DRIFT_REPORT_PATH: output/docs-source-drift.json - run: pnpm check:source-drift - - name: Install Chrome for browser certification run: pnpm exec puppeteer browsers install chrome @@ -70,8 +98,28 @@ jobs: uses: actions/upload-artifact@v4 with: name: docs-certification-${{ github.run_id }} - path: | - output/docs-smoke/ - output/docs-source-drift.json + path: output/docs-smoke/ if-no-files-found: warn retention-days: 30 + + verify: + # Preserve the canonical required-check name while collecting both lanes. + name: Verify documentation release + needs: [freshness, browser] + if: ${{ always() }} + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Install Node.js + uses: actions/setup-node@v4 + with: + node-version: 22 + + - name: Require both evidence jobs to succeed + env: + DOCS_FRESHNESS_RESULT: ${{ needs.freshness.result }} + DOCS_BROWSER_RESULT: ${{ needs.browser.result }} + run: node scripts/docs-release-gate.mjs diff --git a/docs/e2e-source-freshness.md b/docs/e2e-source-freshness.md new file mode 100644 index 0000000..398f10f --- /dev/null +++ b/docs/e2e-source-freshness.md @@ -0,0 +1,74 @@ +# Independent browser evidence and source freshness + +## Why these checks are separate + +Docs run `34664876044`, job `103474663562`, installed dependencies successfully +but failed source freshness before Chrome installation or `pnpm verify`. +That result did not establish a browser regression in the dependency PR. +The upstream drift incident remains #89, not a waiver for application tests. + +The Docs workflow now runs two independent jobs: + +- `freshness` validates the source lock and compares watched source identities. +- `browser` runs the frozen install, Chrome installation, complete `pnpm verify`, + and the existing clean-generated-tree check. + +The final `verify` job retains the check name **Verify documentation release**. +It runs even after either dependency fails and accepts only two literal +`success` results. Missing, skipped, cancelled, pending, and failed results all +fail. There is no continue-on-error path, new test quarantine, or release waiver. +A green browser job alone does not authorize publication or clear #89. + +## Exact snapshot drift semantics + +A review timestamp is metadata, not a Git history boundary. A commit written +before review can be merged afterwards. Re-reading `main` for each path can +also combine different source revisions in one report. + +The detector resolves the watched ref once, verifies that the reviewed commit +is its ancestor, and compares each watched path's identity in the two pinned +Git trees. The report contains the actual `refCommit`, both path identities, +and `verificationMode: git-tree-identity`. The `changes` array is a snapshot +comparison receipt, not an exhaustive commit chronology. Review the linked +exact-commit comparison and affected source before updating public claims. + +Mode, object type, blob/tree SHA, additions, deletions, and ancestor symlink +replacements are significant. Watched directories compare their tree identity. +Unrelated changes outside watched paths do not create drift. A path restored to +exactly its reviewed identity is unchanged for current documentation purposes; +this check is not an audit of all historical changes or deployed behavior. + +The detector does not use the capped comparison file list or date-filtered +commit pages. It traverses non-recursive trees, caches repeated reads, and +refuses incomplete/truncated trees, malformed identities, unproven ancestry, +and paths absent from both snapshots. API denial or incomplete evidence is a +failure, never an empty successful inventory. Raw API error bodies are not +copied to public evidence. + +## Evidence locations and verification + +The Docs workflow retains separate artifacts: + +- `docs-source-drift-`: `docs-source-drift.json`. +- `docs-certification-`: browser/visual evidence under `docs-smoke/`. + +The scheduled drift workflow retains its existing incident behavior and uses +the same detector. `pnpm verify` also runs the dependency-free automation tests +through `check:automation`. + +```sh +node --test scripts/check-source-drift.test.mjs scripts/docs-release-gate.test.mjs +pnpm verify +``` + +The focused suite executes the detector CLI with fixture-controlled GitHub +responses, not live network or deployed data. The gate suite exercises every +pairing of seven job outcomes and verifies the workflow dependency graph. + +## Remaining source review + +This repair does **not** advance `docs/source-lock.json`, change `verifiedAt` or +`verifiedCommit`, review all outstanding upstream contracts, or certify a live +site. Retain #89 until its source-to-page review, affected public-page changes, +full verification, and truthful source-lock update are complete. Do not disable +the canonical gate to land a freshness-only metadata bump. diff --git a/scripts/check-automation-syntax.mjs b/scripts/check-automation-syntax.mjs index acf47d0..6b4fcf7 100644 --- a/scripts/check-automation-syntax.mjs +++ b/scripts/check-automation-syntax.mjs @@ -25,4 +25,14 @@ if (failures.length > 0) { process.exit(1); } +const regressions = spawnSync(process.execPath, [ + '--test', + 'scripts/check-source-drift.test.mjs', + 'scripts/docs-release-gate.test.mjs', +], { cwd: root, stdio: 'inherit' }); +if (regressions.error || regressions.status !== 0) { + console.error('Automation regression tests failed.'); + process.exit(1); +} + console.log(`Automation syntax check passed for ${files.length} JavaScript modules.`); diff --git a/scripts/check-source-drift.mjs b/scripts/check-source-drift.mjs index c224c74..54df77a 100644 --- a/scripts/check-source-drift.mjs +++ b/scripts/check-source-drift.mjs @@ -3,16 +3,13 @@ import { dirname, resolve } from 'node:path'; import process from 'node:process'; const root = resolve(import.meta.dirname, '..'); -const lock = JSON.parse( - await readFile(resolve(root, 'docs/source-lock.json'), 'utf8'), -); const apiBase = process.env.GITHUB_API_URL ?? 'https://api.github.com'; const token = process.env.GITHUB_TOKEN?.trim(); const reportPath = resolve( process.env.DOCS_DRIFT_REPORT_PATH ?? 'output/docs-source-drift.json', ); const maxClockSkewMs = 5 * 60 * 1_000; - +const shaPattern = /^[0-9a-f]{40}$/i; const headers = { Accept: 'application/vnd.github+json', 'X-GitHub-Api-Version': '2022-11-28', @@ -20,14 +17,15 @@ const headers = { ...(token ? { Authorization: `Bearer ${token}` } : {}), }; -async function apiJson(url) { +async function apiJson(path) { + const url = new URL(path, apiBase); const response = await fetch(url, { headers, signal: AbortSignal.timeout(20_000), }); if (!response.ok) { - const body = await response.text(); - throw new Error(`GitHub API ${response.status} for ${url}: ${body.slice(0, 500)}`); + // Do not copy arbitrary API response bodies into public evidence. + throw new Error(`GitHub API ${response.status} for ${url}`); } return response.json(); } @@ -41,92 +39,184 @@ function commitDate(commit) { return commit.commit?.committer?.date ?? commit.commit?.author?.date ?? null; } +function requireCommit(commit, label, expectedSha) { + if ( + !shaPattern.test(commit?.sha ?? '') || + !shaPattern.test(commit?.commit?.tree?.sha ?? '') || + !Number.isFinite(Date.parse(commitDate(commit))) || + (expectedSha && commit.sha.toLowerCase() !== expectedSha.toLowerCase()) + ) { + throw new Error(`${label}: incomplete or mismatched commit identity`); + } + return commit; +} + +// Non-recursive Git trees preserve mode and symlink/gitlink identity. Contents +// responses can dereference symlinks; compare.files is capped at 300 paths. +const treeCache = new Map(); +async function readTree(repo, sha) { + const key = `${repo}/${sha}`; + if (treeCache.has(key)) return treeCache.get(key); + const result = await apiJson(`/repos/${repo}/git/trees/${sha}`); + if ( + result?.sha !== sha || + result.truncated !== false || + !Array.isArray(result.tree) + ) { + throw new Error(`${repo}: incomplete or mismatched tree ${sha}`); + } + const entries = new Map(); + const modes = { + '040000': 'tree', + '100644': 'blob', + '100755': 'blob', + '120000': 'blob', + '160000': 'commit', + }; + for (const entry of result.tree) { + if ( + typeof entry.path !== 'string' || + !entry.path || + entry.path.includes('/') || + entry.path === '.' || + entry.path === '..' || + entries.has(entry.path) || + !Object.hasOwn(modes, entry.mode) || + modes[entry.mode] !== entry.type || + !shaPattern.test(entry.sha ?? '') + ) { + throw new Error(`${repo}: malformed entry in tree ${sha}`); + } + entries.set(entry.path, { + mode: entry.mode, + type: entry.type, + sha: entry.sha, + }); + } + treeCache.set(key, entries); + return entries; +} + +async function pathIdentity(repo, treeSha, path) { + const segments = path.split('/'); + if ( + segments.length > 64 || + segments.some((part) => !part || part === '.' || part === '..') + ) { + throw new Error(`${repo}: invalid watched repository path ${path}`); + } + for (let index = 0; index < segments.length; index += 1) { + const entry = (await readTree(repo, treeSha)).get(segments[index]); + if (!entry) return null; + if (index === segments.length - 1) return entry; + if (entry.type !== 'tree') { + // An ancestor replaced by a symlink/file is drift too, not a directory + // to follow outside the pinned Git tree. + return { ...entry, obstructedAt: segments.slice(0, index + 1).join('/') }; + } + treeSha = entry.sha; + } +} + const sourceResults = []; let changedPathCount = 0; try { + const lock = JSON.parse( + await readFile(resolve(root, 'docs/source-lock.json'), 'utf8'), + ); + if (lock.schemaVersion !== 1 || !Array.isArray(lock.sources) || !lock.sources.length) { + throw new Error('Source lock must contain a non-empty schemaVersion 1 source list'); + } for (const source of lock.sources) { + if ( + !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(source.repo ?? '') || + typeof source.ref !== 'string' || !source.ref || + !shaPattern.test(source.verifiedCommit ?? '') || + !Array.isArray(source.paths) || !source.paths.length || + source.paths.some((path) => typeof path !== 'string' || !path) || + new Set(source.paths).size !== source.paths.length + ) { + throw new Error('Source lock contains an invalid repository, ref, SHA, or path list'); + } const verifiedAtMs = Date.parse(source.verifiedAt); - if (verifiedAtMs > Date.now() + maxClockSkewMs) { - throw new Error( - `${source.id} verifiedAt ${source.verifiedAt} is in the future; refusing a freshness window that can hide changes`, - ); + if (!Number.isFinite(verifiedAtMs) || verifiedAtMs > Date.now() + maxClockSkewMs) { + throw new Error(`${source.id}: invalid or future verifiedAt`); } - - const verifiedCommitUrl = new URL( - `/repos/${source.repo}/commits/${source.verifiedCommit}`, - apiBase, + const prefix = `/repos/${source.repo}`; + const verified = requireCommit( + await apiJson(`${prefix}/commits/${source.verifiedCommit}`), + source.id, + source.verifiedCommit, ); - const verifiedCommit = await apiJson(verifiedCommitUrl); - const verifiedCommitDate = commitDate(verifiedCommit); - if (!verifiedCommitDate) { - throw new Error(`${source.id} verified commit has no GitHub commit timestamp`); - } - if (Date.parse(verifiedCommitDate) > verifiedAtMs) { - throw new Error( - `${source.id} verifiedAt ${source.verifiedAt} predates verified commit ${source.verifiedCommit} at ${verifiedCommitDate}`, - ); + if (Date.parse(commitDate(verified)) > verifiedAtMs) { + throw new Error(`${source.id}: verifiedAt predates the verified commit`); } - const compareUrl = new URL( - `/repos/${source.repo}/compare/${source.verifiedCommit}...${encodeURIComponent(source.ref)}`, - apiBase, + // Resolve the mutable ref once. Every remaining read is bound to these + // immutable commit/tree identities, never to timestamps or a moving main. + const target = requireCommit( + await apiJson(`${prefix}/commits/${encodeURIComponent(source.ref)}`), + source.id, + ); + const comparison = await apiJson( + `${prefix}/compare/${verified.sha}...${target.sha}?per_page=1`, ); - const comparison = await apiJson(compareUrl); - if (!['ahead', 'identical'].includes(comparison.status)) { - throw new Error( - `${source.id} verified commit is not an ancestor of ${source.ref}; compare status is ${comparison.status}`, - ); + if ( + !['ahead', 'identical'].includes(comparison.status) || + comparison.base_commit?.sha !== verified.sha || + comparison.merge_base_commit?.sha !== verified.sha || + (comparison.status === 'identical') !== (verified.sha === target.sha) + ) { + throw new Error(`${source.id}: verified commit is not the pinned target's proven ancestor`); } const pathResults = []; for (const path of source.paths) { - const url = new URL(`/repos/${source.repo}/commits`, apiBase); - url.searchParams.set('sha', source.ref); - url.searchParams.set('path', path); - url.searchParams.set('since', source.verifiedAt); - url.searchParams.set('per_page', '100'); - const commits = await apiJson(url); - const changes = commits - .filter((commit) => commit.sha !== source.verifiedCommit) - .map((commit) => ({ - sha: commit.sha, - date: commitDate(commit), - summary: commit.commit?.message?.split('\n')[0] ?? '', - url: commit.html_url, - })); - if (changes.length > 0) changedPathCount += 1; - pathResults.push({ path, changes }); + const before = await pathIdentity(source.repo, verified.commit.tree.sha, path); + const after = await pathIdentity(source.repo, target.commit.tree.sha, path); + if (before === null && after === null) { + throw new Error(`${source.id}: watched path absent from both pinned trees: ${path}`); + } + const changed = JSON.stringify(before) !== JSON.stringify(after); + if (changed) changedPathCount += 1; + pathResults.push({ + path, + before, + after, + changes: changed ? [{ + sha: target.sha, + date: commitDate(target), + summary: 'Watched path identity differs from the verified Git tree', + url: `https://github.com/${source.repo}/compare/${verified.sha}...${target.sha}`, + }] : [], + }); } sourceResults.push({ id: source.id, repo: source.repo, ref: source.ref, - refCommit: comparison.head_commit?.sha ?? null, + refCommit: target.sha, verifiedAt: source.verifiedAt, - verifiedCommit: source.verifiedCommit, - verifiedCommitDate, + verifiedCommit: verified.sha, + verifiedCommitDate: commitDate(verified), + verificationMode: 'git-tree-identity', sections: source.sections, paths: pathResults, }); } - - const report = { + await writeReport({ ok: changedPathCount === 0, checkedAt: new Date().toISOString(), changedPathCount, sources: sourceResults, - }; - await writeReport(report); - + }); if (changedPathCount > 0) { - console.error( - `Upstream contract drift detected in ${changedPathCount} watched path(s). Review output at ${reportPath}.`, - ); - process.exit(1); + console.error(`Upstream contract drift detected in ${changedPathCount} watched path(s). Review output at ${reportPath}.`); + process.exitCode = 1; + } else { + console.log(`No upstream contract drift across ${sourceResults.length} source(s).`); } - - console.log(`No upstream contract drift across ${sourceResults.length} source(s).`); } catch (error) { const message = error instanceof Error ? error.message : String(error); await writeReport({ @@ -136,5 +226,5 @@ try { sources: sourceResults, }); console.error(message); - process.exit(1); + process.exitCode = 1; } diff --git a/scripts/check-source-drift.test.mjs b/scripts/check-source-drift.test.mjs new file mode 100644 index 0000000..edfd983 --- /dev/null +++ b/scripts/check-source-drift.test.mjs @@ -0,0 +1,255 @@ +import assert from 'node:assert/strict'; +import { mkdtempSync, readFileSync, writeFileSync, mkdirSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { spawnSync } from 'node:child_process'; +import test from 'node:test'; + +const A = 'a'.repeat(40); +const B = 'b'.repeat(40); +const C = 'c'.repeat(40); +const D = 'd'.repeat(40); +const E = 'e'.repeat(40); +const F = 'f'.repeat(40); +const prefix = '/repos/OpenCoven/example'; +const script = readFileSync(resolve(import.meta.dirname, 'check-source-drift.mjs'), 'utf8'); +const entry = (path, sha = E, mode = '100644', type = 'blob') => ({ path, sha, mode, type }); +const commit = (sha, tree, date = '2026-09-01T00:00:00Z') => ({ + sha, commit: { tree: { sha: tree }, committer: { date } }, +}); +const tree = (sha, entries) => ({ sha, truncated: false, tree: entries }); + +function fixture() { + const lock = { schemaVersion: 1, sources: [{ + id: 'runtime', repo: 'OpenCoven/example', ref: 'main', + verifiedCommit: A, verifiedAt: '2026-09-02T00:00:00Z', + sections: ['guide'], paths: ['contract.md'], + }] }; + const comparison = { status: 'ahead', base_commit: { sha: A }, merge_base_commit: { sha: A } }; + const replies = { + [`${prefix}/commits/${A}`]: commit(A, B), + [`${prefix}/commits/main`]: commit(C, D, '2026-09-03T00:00:00Z'), + [`${prefix}/compare/${A}...${C}?per_page=1`]: comparison, + [`${prefix}/git/trees/${B}`]: tree(B, [entry('contract.md')]), + [`${prefix}/git/trees/${D}`]: tree(D, [entry('contract.md')]), + // Also model the old CLI's time-filtered API calls. Empty date-filtered + // history is NOT proof that the pinned file still has its reviewed bytes. + [`${prefix}/compare/${A}...main`]: comparison, + [`${prefix}/commits?sha=main&path=contract.md&since=2026-09-02T00%3A00%3A00Z&per_page=100`]: [], + }; + return { lock, replies }; +} + +function run(input) { + const dir = mkdtempSync(join(tmpdir(), 'docs-source-drift-')); + try { + mkdirSync(join(dir, 'scripts')); + mkdirSync(join(dir, 'docs')); + writeFileSync(join(dir, 'scripts/check-source-drift.mjs'), script); + writeFileSync(join(dir, 'docs/source-lock.json'), JSON.stringify(input.lock)); + writeFileSync(join(dir, 'fixture.json'), JSON.stringify(input.replies)); + writeFileSync(join(dir, 'mock.mjs'), ` + import { readFileSync, appendFileSync } from 'node:fs'; + const replies = JSON.parse(readFileSync('fixture.json', 'utf8')); + Date.now = () => Date.parse('2026-09-14T00:00:00Z'); + globalThis.fetch = async (input) => { + const u = new URL(input); + const key = u.pathname + u.search; + appendFileSync('requests.jsonl', JSON.stringify(key) + '\\n'); + if (!Object.hasOwn(replies, key)) throw new Error('Unexpected request: ' + key); + const value = replies[key]; + if (value && value.__throw) throw new Error(value.__throw); + if (value && value.__status) return new Response('DO-NOT-LOG-REMOTE-BODY', { status: value.__status }); + return new Response(JSON.stringify(value)); + }; + `); + const result = spawnSync(process.execPath, ['--import', './mock.mjs', 'scripts/check-source-drift.mjs'], { + cwd: dir, + encoding: 'utf8', + timeout: 5_000, + env: { ...process.env, GITHUB_TOKEN: '', GITHUB_API_URL: 'https://api.github.com', DOCS_DRIFT_REPORT_PATH: 'report.json' }, + }); + assert.ifError(result.error); + const report = JSON.parse(readFileSync(join(dir, 'report.json'), 'utf8')); + let requests = []; + try { requests = readFileSync(join(dir, 'requests.jsonl'), 'utf8').trim().split('\n').filter(Boolean).map(JSON.parse); } catch { /* Lock rejection can precede any request. */ } + return { ...result, report, requests }; + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +function expectDrift(input) { + const result = run(input); + assert.equal(result.status, 1, result.stderr); + assert.equal(result.report.ok, false); + assert.equal(result.report.error, undefined, result.report.error); + assert.equal(result.report.changedPathCount, 1); + assert.equal(result.report.sources[0].refCommit, C); + return result; +} +function expectFailure(input) { + const result = run(input); + assert.equal(result.status, 1, result.stderr); + assert.equal(result.report.ok, false); + assert.equal(typeof result.report.error, 'string'); + return result; +} + +test('unchanged watched bytes pass even if unrelated files changed', () => { + const f = fixture(); + f.replies[`${prefix}/git/trees/${D}`].tree.push(entry('unrelated.md', F)); + const result = run(f); + assert.equal(result.status, 0, result.stderr); + assert.equal(result.report.changedPathCount, 0); + assert.equal(result.report.sources[0].refCommit, C); +}); + +test('late-merged old-dated change cannot hide behind verifiedAt', () => { + const f = fixture(); + f.replies[`${prefix}/commits/main`].commit.committer.date = '2026-09-01T12:00:00Z'; + f.replies[`${prefix}/git/trees/${D}`].tree[0].sha = F; + const result = expectDrift(f); + assert.ok(result.requests.every((request) => !request.includes('since='))); +}); + +test('a moving ref is sampled once and never queried for path history', () => { + const f = fixture(); + f.replies[`${prefix}/git/trees/${D}`].tree[0].sha = F; + const result = expectDrift(f); + assert.equal(result.requests.filter((request) => request === `${prefix}/commits/main`).length, 1); + assert.ok(result.requests.includes(`${prefix}/compare/${A}...${C}?per_page=1`)); + assert.ok(!result.requests.some((request) => request.includes('...main') || request.includes('/commits?'))); +}); + +test('deletion is drift', () => { + const f = fixture(); + f.replies[`${prefix}/git/trees/${D}`].tree = []; + assert.equal(expectDrift(f).report.sources[0].paths[0].after, null); +}); + +test('addition is drift', () => { + const f = fixture(); + f.replies[`${prefix}/git/trees/${B}`].tree = []; + assert.equal(expectDrift(f).report.sources[0].paths[0].before, null); +}); + +for (const mode of ['100755', '120000']) { + test(`mode change to ${mode} is drift even at the same blob SHA`, () => { + const f = fixture(); + f.replies[`${prefix}/git/trees/${D}`].tree[0].mode = mode; + expectDrift(f); + }); +} + +test('missing from both snapshots is an invalid watch, not success', () => { + const f = fixture(); + f.replies[`${prefix}/git/trees/${B}`].tree = []; + f.replies[`${prefix}/git/trees/${D}`].tree = []; + expectFailure(f); +}); + +for (const variant of ['truncated', 'missing-truncated', 'wrong-sha', 'duplicate', 'malformed']) { + test(`incomplete tree ${variant} fails closed`, () => { + const f = fixture(); + const t = f.replies[`${prefix}/git/trees/${D}`]; + if (variant === 'truncated') t.truncated = true; + if (variant === 'missing-truncated') delete t.truncated; + if (variant === 'wrong-sha') t.sha = B; + if (variant === 'duplicate') t.tree.push(t.tree[0]); + if (variant === 'malformed') t.tree[0].sha = 'bad'; + expectFailure(f); + }); +} + +test('comparison file-list truncation cannot conceal a watched path', () => { + const f = fixture(); + f.replies[`${prefix}/compare/${A}...${C}?per_page=1`].files = Array.from({ length: 300 }, (_, i) => ({ filename: `other-${i}` })); + f.replies[`${prefix}/git/trees/${D}`].tree[0].sha = F; + expectDrift(f); +}); + +for (const status of ['behind', 'diverged', 'unknown']) { + test(`ancestry ${status} fails closed`, () => { + const f = fixture(); + f.replies[`${prefix}/compare/${A}...${C}?per_page=1`].status = status; + expectFailure(f); + }); +} + +test('ancestry must bind the verified base and merge base', () => { + const f = fixture(); + f.replies[`${prefix}/compare/${A}...${C}?per_page=1`].merge_base_commit.sha = C; + expectFailure(f); +}); + +test('exact identical commit passes without inventing a new verification boundary', () => { + const f = fixture(); + f.replies[`${prefix}/commits/main`] = commit(A, B); + f.replies[`${prefix}/compare/${A}...${A}?per_page=1`] = { status: 'identical', base_commit: { sha: A }, merge_base_commit: { sha: A } }; + const result = run(f); + assert.equal(result.status, 0, result.stderr); + assert.equal(result.report.sources[0].refCommit, A); + assert.equal(result.requests.filter((request) => request.includes('/git/trees/')).length, 1); +}); + +test('nested trees are read without recursion and shared tree reads are cached', () => { + const f = fixture(); + f.lock.sources[0].paths = ['docs/one.md', 'docs/two.md']; + f.replies[`${prefix}/git/trees/${B}`].tree = [entry('docs', E, '040000', 'tree')]; + f.replies[`${prefix}/git/trees/${D}`].tree = [entry('docs', F, '040000', 'tree')]; + f.replies[`${prefix}/git/trees/${E}`] = tree(E, [entry('one.md', A), entry('two.md', B)]); + f.replies[`${prefix}/git/trees/${F}`] = tree(F, [entry('one.md', A), entry('two.md', C)]); + const result = expectDrift(f); + assert.equal(result.requests.filter((request) => request.includes('/git/trees/')).length, 4); + assert.ok(result.requests.every((request) => !request.includes('recursive'))); +}); + +test('ancestor symlink replacement is not followed', () => { + const f = fixture(); + f.lock.sources[0].paths = ['docs/one.md']; + f.replies[`${prefix}/git/trees/${B}`].tree = [entry('docs', E, '040000', 'tree')]; + f.replies[`${prefix}/git/trees/${D}`].tree = [entry('docs', F, '120000')]; + f.replies[`${prefix}/git/trees/${E}`] = tree(E, [entry('one.md', A)]); + const result = expectDrift(f); + assert.equal(result.report.sources[0].paths[0].after.obstructedAt, 'docs'); + assert.ok(!result.requests.includes(`${prefix}/git/trees/${F}`)); +}); + +for (const status of [403, 404, 429, 500]) { + test(`HTTP ${status} retains a failing report without the response body`, () => { + const f = fixture(); + f.replies[`${prefix}/git/trees/${D}`] = { __status: status }; + const result = expectFailure(f); + assert.ok(!JSON.stringify(result.report).includes('DO-NOT-LOG-REMOTE-BODY')); + assert.ok(!result.stderr.includes('DO-NOT-LOG-REMOTE-BODY')); + }); +} + +test('network timeout remains a failure with a report', () => { + const f = fixture(); + f.replies[`${prefix}/git/trees/${D}`] = { __throw: 'bounded request timeout' }; + expectFailure(f); +}); + +for (const value of ['not-a-date', '2027-01-01T00:00:00Z', '2026-08-01T00:00:00Z']) { + test(`invalid review timestamp ${value} fails closed`, () => { + const f = fixture(); + f.lock.sources[0].verifiedAt = value; + expectFailure(f); + }); +} + +for (const path of ['../contract.md', '/contract.md', 'docs//contract.md']) { + test(`invalid path ${path} fails closed`, () => { + const f = fixture(); + f.lock.sources[0].paths = [path]; + expectFailure(f); + }); +} + +test('empty lock cannot yield a vacuous green result', () => { + const f = fixture(); + f.lock.sources = []; + expectFailure(f); +}); diff --git a/scripts/docs-release-gate.mjs b/scripts/docs-release-gate.mjs new file mode 100644 index 0000000..f9089d5 --- /dev/null +++ b/scripts/docs-release-gate.mjs @@ -0,0 +1,17 @@ +import process from 'node:process'; + +// Keep the canonical required check red unless BOTH independent evidence jobs +// succeeded. Missing, skipped, cancelled, and unknown outcomes are not passes. +const results = [ + ['upstream source freshness', process.env.DOCS_FRESHNESS_RESULT], + ['documentation/browser certification', process.env.DOCS_BROWSER_RESULT], +]; +const failures = results.filter(([, result]) => result !== 'success'); +if (failures.length) { + for (const [name, result] of failures) { + console.error(`${name}: ${result ?? 'missing'}`); + } + process.exitCode = 1; +} else { + console.log('Documentation release requires and has both successful evidence jobs.'); +} diff --git a/scripts/docs-release-gate.test.mjs b/scripts/docs-release-gate.test.mjs new file mode 100644 index 0000000..43c2643 --- /dev/null +++ b/scripts/docs-release-gate.test.mjs @@ -0,0 +1,41 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { spawnSync } from 'node:child_process'; +import test from 'node:test'; + +const gate = resolve(import.meta.dirname, 'docs-release-gate.mjs'); +const outcomes = ['success', 'failure', 'cancelled', 'skipped', 'pending', '', undefined]; +for (const freshness of outcomes) { + for (const browser of outcomes) { + test(`release gate: freshness=${freshness}, browser=${browser}`, () => { + const env = { ...process.env }; + delete env.DOCS_FRESHNESS_RESULT; + delete env.DOCS_BROWSER_RESULT; + if (freshness !== undefined) env.DOCS_FRESHNESS_RESULT = freshness; + if (browser !== undefined) env.DOCS_BROWSER_RESULT = browser; + const result = spawnSync(process.execPath, [gate], { env, encoding: 'utf8', timeout: 5_000 }); + assert.ifError(result.error); + assert.equal(result.status, freshness === 'success' && browser === 'success' ? 0 : 1); + }); + } +} + +test('browser evidence does not depend on freshness; canonical rollup requires both', () => { + const workflow = readFileSync(resolve(import.meta.dirname, '../.github/workflows/docs.yml'), 'utf8'); + const jobs = Object.fromEntries([...workflow.slice(workflow.indexOf('jobs:\n') + 6).matchAll(/^ ([a-z-]+):\n([\s\S]*?)(?=^ [a-z-]+:\n|$(?![\s\S]))/gm)].map(([, id, body]) => [id, body])); + assert.deepEqual(Object.keys(jobs).sort(), ['browser', 'freshness', 'verify']); + assert.doesNotMatch(jobs.browser, /^ (?:needs|if):/m); + assert.match(jobs.browser, /run: pnpm verify/); + assert.match(jobs.browser, /pnpm install --frozen-lockfile/); + assert.match(jobs.browser, /Confirm clean generated tree/); + assert.match(jobs.freshness, /node scripts\/check-source-lock\.mjs/); + assert.match(jobs.freshness, /node scripts\/check-source-drift\.mjs/); + assert.match(jobs.verify, /name: Verify documentation release/); + assert.match(jobs.verify, /needs: \[freshness, browser\]/); + assert.match(jobs.verify, /if: \$\{\{ always\(\) \}\}/); + assert.match(jobs.verify, /DOCS_FRESHNESS_RESULT: \$\{\{ needs\.freshness\.result \}\}/); + assert.match(jobs.verify, /DOCS_BROWSER_RESULT: \$\{\{ needs\.browser\.result \}\}/); + assert.match(jobs.verify, /run: node scripts\/docs-release-gate\.mjs/); + assert.doesNotMatch(workflow, /continue-on-error/); +});