From 26bb2858be18772ce38b3847d8271035da72d1fe Mon Sep 17 00:00:00 2001 From: Mark Beacom Date: Wed, 5 Aug 2026 19:32:27 -0400 Subject: [PATCH 1/2] =?UTF-8?q?feat(catalog):=20feature=20010=20Phase=20F?= =?UTF-8?q?=20=E2=80=94=20clause-5=20step=20(b)=20post-output=20comparison?= =?UTF-8?q?=20(T086a,=20T087=E2=80=93T092)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR-0020 clause 5 requires two distinct steps. Step (a) — the pre-output freeze and its independent audit — landed with Barrier B. This is step (b): run the assembled generator over the frozen accept corpus and diff derived ownership against the frozen expectations at zero false positives and zero false negatives. Result: PASS. 24 expected entities, 0 false positives, 0 false negatives, 0 other mismatches. The frozen trees were not touched, and their hashes are unchanged from the values T024 recorded before any generator existed. T086a (new, prerequisite) — vendor the accept corpus. Phase E discovered the corpus was not materialized: the freeze records metadata, never descriptor files, and evidence/README.md §4 requires it stay that way. The 24 selected descriptors are vendored verbatim from backstage/community-plugins at the pinned commit into specs/010-catalog-backstage/corpus/, each verified by recomputing its git blob id from the bytes that arrived and matching it against the id the pinned tree records, with the pin cross-checked against the freeze's own 156-descriptor-file figure. Any mismatch aborts and writes nothing. The vendored bytes are pristine: no adrkit.io/owned-paths anywhere in the tree. The maintainer-authored overlay stays in overlay.json and is applied at generation time into a temporary directory that is deleted, so clause 5's "otherwise unmodified" stays provable by digest and data-model.md §10's upstream/maintainer boundary stays legible by inspection. Acquisition is a one-time step and is not reachable from the generator or the harness, asserted by scanning import specifiers rather than promised in prose. T087 — scripts/compare-accept-corpus.ts, authored after the freeze and its audit (R5 mechanism 3), with the provenance record that carries the ordering claim. T088 — the diff itself. The comparison reads both frozen artifacts and refuses to run if they disagree. Entities outside the expectation set are findings only if they derive a non-empty path set: the 24 files hold 25 entity documents, and a descriptor file count is not an entity document count. T089 — observed failing. One character in one overlay value (cli → clx) yields exactly one false negative, one false positive, and the corpus-wide union mismatch; restoring returns PASS. Retained at negative-cases/comparison-mismatch/. T090 — step (b) recomputes the frozen hashes rather than trusting them, records hashes that are its own (corpus, overlaid inputs, envelope), and renders its own verdict. It inherits nothing from step (a). T091 — the prohibition guard. Every frozen hash is compared against the value the Barrier B checkpoint recorded, and the amended-then-re-signed case is driven to show why self-consistency alone would not catch it. T092 — reporting honesty. A digest establishes integrity; this comparison establishes agreement with a maintainer-authored expectation set. Neither establishes that the adapter is correct, and every Phase F artifact is scanned for affirmative correctness and rung overclaims. T086 stays unchecked: its SC-009 limb-2 record is corrected to the new fact, but wiring a corpus pass into that close-out is T086's own remaining work. ADR-0014 rung 1 only. No release is scheduled, prepared, or implied. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../catalog-backstage/test/sc-009.test.ts | 61 +- scripts/check-freeze-hashes.test.ts | 116 ++ scripts/compare-accept-corpus.test.ts | 645 +++++++++ scripts/compare-accept-corpus.ts | 1195 +++++++++++++++++ scripts/vendor-accept-corpus.test.ts | 347 +++++ scripts/vendor-accept-corpus.ts | 398 ++++++ specs/010-catalog-backstage/corpus/README.md | 81 ++ .../corpus/VENDOR-MANIFEST.json | 171 +++ .../acr/plugins/acr/catalog-info.yaml | 51 + .../adr/plugins/adr-backend/catalog-info.yaml | 9 + .../plugins/agent-forge/catalog-info.yaml | 10 + .../airbrake-backend/catalog-info.yaml | 9 + .../allure/plugins/allure/catalog-info.yaml | 10 + .../analytics-module-ga4/catalog-info.yaml | 9 + .../plugins/apache-airflow/catalog-info.yaml | 9 + .../plugins/apollo-explorer/catalog-info.yaml | 9 + .../azure-devops-common/catalog-info.yaml | 9 + .../azure-sites-backend/catalog-info.yaml | 9 + .../plugins/badges-backend/catalog-info.yaml | 10 + .../plugins/bazaar-backend/catalog-info.yaml | 9 + .../bitbucket-pull-requests/catalog-info.yaml | 13 + .../bitrise/plugins/bitrise/catalog-info.yaml | 10 + .../examples/component/catalog-info.yaml | 23 + .../catalog-info.yaml | 10 + .../checkmarx-backend/catalog-info.yaml | 9 + .../catalog-info.yaml | 10 + .../plugins/cloudbuild/catalog-info.yaml | 10 + .../plugins/code-climate/catalog-info.yaml | 9 + .../code-coverage-backend/catalog-info.yaml | 10 + .../plugins/codescene/catalog-info.yaml | 9 + .../plugins/copilot-backend/catalog-info.yaml | 9 + .../cost-insights-common/catalog-info.yaml | 10 + .../010-catalog-backstage/evidence/README.md | 34 +- .../evidence/comparison/README.md | 55 + .../evidence/comparison/diff-report.json | 300 +++++ .../comparison/expectations-unchanged.json | 33 + .../evidence/comparison/harness-provenance.md | 99 ++ .../evidence/comparison/reporting-honesty.md | 94 ++ .../evidence/comparison/step-b-record.json | 94 ++ .../comparison-mismatch/README.md | 90 ++ .../comparison-mismatch/observed-fail.txt | 6 + .../comparison-mismatch/restored.observed.txt | 2 + specs/010-catalog-backstage/tasks.md | 44 +- 43 files changed, 4118 insertions(+), 32 deletions(-) create mode 100644 scripts/compare-accept-corpus.test.ts create mode 100644 scripts/compare-accept-corpus.ts create mode 100644 scripts/vendor-accept-corpus.test.ts create mode 100644 scripts/vendor-accept-corpus.ts create mode 100644 specs/010-catalog-backstage/corpus/README.md create mode 100644 specs/010-catalog-backstage/corpus/VENDOR-MANIFEST.json create mode 100644 specs/010-catalog-backstage/corpus/workspaces/acr/plugins/acr/catalog-info.yaml create mode 100644 specs/010-catalog-backstage/corpus/workspaces/adr/plugins/adr-backend/catalog-info.yaml create mode 100644 specs/010-catalog-backstage/corpus/workspaces/agent-forge/plugins/agent-forge/catalog-info.yaml create mode 100644 specs/010-catalog-backstage/corpus/workspaces/airbrake/plugins/airbrake-backend/catalog-info.yaml create mode 100644 specs/010-catalog-backstage/corpus/workspaces/allure/plugins/allure/catalog-info.yaml create mode 100644 specs/010-catalog-backstage/corpus/workspaces/analytics/plugins/analytics-module-ga4/catalog-info.yaml create mode 100644 specs/010-catalog-backstage/corpus/workspaces/apache-airflow/plugins/apache-airflow/catalog-info.yaml create mode 100644 specs/010-catalog-backstage/corpus/workspaces/apollo-explorer/plugins/apollo-explorer/catalog-info.yaml create mode 100644 specs/010-catalog-backstage/corpus/workspaces/azure-devops/plugins/azure-devops-common/catalog-info.yaml create mode 100644 specs/010-catalog-backstage/corpus/workspaces/azure-sites/plugins/azure-sites-backend/catalog-info.yaml create mode 100644 specs/010-catalog-backstage/corpus/workspaces/badges/plugins/badges-backend/catalog-info.yaml create mode 100644 specs/010-catalog-backstage/corpus/workspaces/bazaar/plugins/bazaar-backend/catalog-info.yaml create mode 100644 specs/010-catalog-backstage/corpus/workspaces/bitbucket-pull-requests/catalog-info.yaml create mode 100644 specs/010-catalog-backstage/corpus/workspaces/bitrise/plugins/bitrise/catalog-info.yaml create mode 100644 specs/010-catalog-backstage/corpus/workspaces/bookmarks/plugins/bookmarks/examples/component/catalog-info.yaml create mode 100644 specs/010-catalog-backstage/corpus/workspaces/catalog/plugins/catalog-backend-module-codeowners/catalog-info.yaml create mode 100644 specs/010-catalog-backstage/corpus/workspaces/checkmarx/plugins/checkmarx-backend/catalog-info.yaml create mode 100644 specs/010-catalog-backstage/corpus/workspaces/cicd-statistics/plugins/cicd-statistics-module-buildkite/catalog-info.yaml create mode 100644 specs/010-catalog-backstage/corpus/workspaces/cloudbuild/plugins/cloudbuild/catalog-info.yaml create mode 100644 specs/010-catalog-backstage/corpus/workspaces/code-climate/plugins/code-climate/catalog-info.yaml create mode 100644 specs/010-catalog-backstage/corpus/workspaces/code-coverage/plugins/code-coverage-backend/catalog-info.yaml create mode 100644 specs/010-catalog-backstage/corpus/workspaces/codescene/plugins/codescene/catalog-info.yaml create mode 100644 specs/010-catalog-backstage/corpus/workspaces/copilot/plugins/copilot-backend/catalog-info.yaml create mode 100644 specs/010-catalog-backstage/corpus/workspaces/cost-insights/plugins/cost-insights-common/catalog-info.yaml create mode 100644 specs/010-catalog-backstage/evidence/comparison/README.md create mode 100644 specs/010-catalog-backstage/evidence/comparison/diff-report.json create mode 100644 specs/010-catalog-backstage/evidence/comparison/expectations-unchanged.json create mode 100644 specs/010-catalog-backstage/evidence/comparison/harness-provenance.md create mode 100644 specs/010-catalog-backstage/evidence/comparison/reporting-honesty.md create mode 100644 specs/010-catalog-backstage/evidence/comparison/step-b-record.json create mode 100644 specs/010-catalog-backstage/evidence/negative-cases/comparison-mismatch/README.md create mode 100644 specs/010-catalog-backstage/evidence/negative-cases/comparison-mismatch/observed-fail.txt create mode 100644 specs/010-catalog-backstage/evidence/negative-cases/comparison-mismatch/restored.observed.txt diff --git a/packages/adapters/catalog-backstage/test/sc-009.test.ts b/packages/adapters/catalog-backstage/test/sc-009.test.ts index 510f2aba..5a393165 100644 --- a/packages/adapters/catalog-backstage/test/sc-009.test.ts +++ b/packages/adapters/catalog-backstage/test/sc-009.test.ts @@ -20,24 +20,30 @@ * commit `92e9e4e09c76cc57f3475029b73e5ec84498a459` * (`evidence/accept-corpus-freeze/accept-corpus-freeze.json`). * - * **Those descriptor files are not present in this repository.** The freeze records the - * corpus's *metadata* — source paths, canonical ids, the overlay values, the expected - * paths — but not the descriptors themselves. Every fixture in this file is - * maintainer-authored, so none of them meets clause 5's conditions, and asserting - * otherwise would present a synthetic corpus as a third-party one. + * **When Phase E wrote this file those descriptor files were not present in this + * repository.** The freeze records the corpus's *metadata* — source paths, canonical ids, + * the overlay values, the expected paths — but not the descriptors themselves. Every + * fixture in this file is maintainer-authored, so none of them meets clause 5's + * conditions, and asserting otherwise would present a synthetic corpus as a third-party + * one. * * Reconstructing descriptors from the freeze was considered and rejected twice over: the * reconstruction would be maintainer-authored (so still not clause-5 conforming), and * feeding a corpus derived from the freeze back in would make the input a function of * the expectations — which is the circularity Barrier B exists to prevent. * - * Materializing the pinned corpus is **Phase F's** concern: T088 diffs "every annotated - * entity in the frozen accept corpus" against the frozen expectations and cannot run - * without it either. {@link acceptCorpusIsMaterialized} below is an executable record of - * the gap: it fails the day someone vendors the corpus, which is the day this limb - * becomes dischargeable and this file should be completed. + * **That gap is now closed.** T086a vendored the 24 descriptors verbatim from the pin + * into `specs/010-catalog-backstage/corpus/`, verified against the content address the + * pinned commit fixes (`scripts/vendor-accept-corpus.ts`). The block below is updated to + * record the new fact rather than left asserting a false one — but it is updated *only* + * to that extent. * - * T086 is therefore left unchecked in `tasks.md`, with this reason. + * **Limb 2 is still not discharged here, and T086 stays unchecked.** Discharging it means + * asserting, in this file, that a pass over the frozen accept corpus produces a populated + * envelope. That pass is Phase F's (`scripts/compare-accept-corpus.ts`, T088), and wiring + * its result into SC-009's close-out is T086's own remaining work — outside the scope of + * the session that vendored the corpus. What the block below now records is the state of + * the repository, which is the fact it was always about. */ import { afterAll, beforeAll, describe, expect, test } from 'bun:test'; @@ -66,10 +72,11 @@ afterAll(async () => { }); /** - * Whether the pinned accept corpus's descriptor files exist anywhere in this repository. + * Whether the pinned accept corpus's descriptor files exist in this repository. * - * Looks for a vendored corpus directory rather than for the freeze metadata, which is - * committed and is not the corpus. + * Looks for the vendored corpus directory rather than for the freeze metadata, which is + * committed and is not the corpus. Phase E wrote this to record an absence; T086a made + * it a presence, so it now reports which. */ function acceptCorpusIsMaterialized(): boolean { const candidates = [ @@ -186,17 +193,20 @@ describe('T086 / SC-009 limb 1 — every pass yields an envelope or a clean reje }); }); -describe('T086 / SC-009 limb 2 — NOT discharged, and why', () => { - test('the frozen accept corpus is not materialized in this repository', () => { - // An executable record of the gap rather than a prose note. When someone vendors - // the pinned corpus this test fails, which is the signal that limb 2 has become - // dischargeable and this file should be completed. - expect(acceptCorpusIsMaterialized()).toBe(false); +describe('T086 / SC-009 limb 2 — still NOT discharged here, and why', () => { + test('the frozen accept corpus is now materialized in this repository (T086a)', () => { + // Phase E recorded the opposite, because it was true then. T086a vendored the 24 + // pinned descriptors, so the fact changed and this record changed with it. What has + // NOT changed is that limb 2 is undischarged: a populated envelope over this corpus + // is produced by Phase F's harness, and wiring that result into SC-009's close-out + // is T086's remaining work. + expect(acceptCorpusIsMaterialized()).toBe(true); }); - test('the freeze records the corpus metadata, not the descriptors', async () => { - // Evidence for the claim above: what is committed is the selection basis, the - // overlay, and the expected paths — never a descriptor file. + test('the freeze still records the corpus metadata, not the descriptors', async () => { + // Unchanged and load-bearing: the descriptors are vendored OUTSIDE the freeze tree. + // evidence/README.md §4 requires the freeze tree to carry no corpus the generator + // could read, because R5 mechanism 1 (input absence) depends on it. const freeze = (await Bun.file( join(REPO_ROOT, 'specs', '010-catalog-backstage', 'evidence', 'accept-corpus-freeze', 'accept-corpus-freeze.json'), ).json()) as Record; @@ -209,10 +219,11 @@ describe('T086 / SC-009 limb 2 — NOT discharged, and why', () => { expect(Object.hasOwn(freeze, 'sources')).toBe(false); }); - test('every fixture in this suite is maintainer-authored, so none meets clause 5', async () => { + test('every fixture in THIS suite is still maintainer-authored, so none meets clause 5', async () => { // Stated as an assertion so no reader mistakes a passing accept case above for a // clause-5 conforming pass. Clause 5 requires descriptors "authored upstream and - // otherwise unmodified"; these were written by this test file. + // otherwise unmodified"; these were written by this test file. The vendored corpus + // is not used here — it is used by scripts/compare-accept-corpus.ts. const { request } = await stage( checkout, { 'sc009-prov/catalog-info.yaml': validDescriptor('sconineprov', '["packages/p/**"]') }, diff --git a/scripts/check-freeze-hashes.test.ts b/scripts/check-freeze-hashes.test.ts index 9684c33c..0aaa94aa 100644 --- a/scripts/check-freeze-hashes.test.ts +++ b/scripts/check-freeze-hashes.test.ts @@ -2,6 +2,7 @@ import { afterEach, describe, expect, test } from 'bun:test'; import { cp, readFile, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; import { cleanupTestDir, resetTestDir } from '../packages/core/test/helpers.ts'; +import { canonicalHash } from './audit-oracle-freeze.ts'; import { checkFreezeHashes, REASON_DRIFT } from './check-freeze-hashes.ts'; const DIR_NAME = 'check-freeze-hashes'; @@ -31,6 +32,121 @@ describe('T022 — freeze-hash drift check (R5 mechanism 2)', () => { }); }); +describe('T091 — the clause-5 prohibition: expectations are never amended to fit output', () => { + /** + * Phase B's hash values, read from the Barrier B checkpoint. + * + * The checkpoint was written by the independent auditor session — the author of the + * T019 audit procedure, and *not* the author of the T014–T018 freeze — before any + * generator existed. So the baseline is not a value Phase E or Phase F could have + * chosen, which is the whole point of comparing against it. + */ + async function phaseBHashes(): Promise> { + const checkpoint = JSON.parse( + await readFile(join(LIVE_EVIDENCE, 'barrier-b-checkpoint.json'), 'utf8'), + ) as { + confirmations: { mechanism2_hashMatch: { recordedHashes: Record } }; + }; + return checkpoint.confirmations.mechanism2_hashMatch.recordedHashes; + } + + test('every frozen hash is unchanged from its Phase B value, across all of E and F', async () => { + // ADR-0020 clause 5: "the expectations are never amended to fit the output". A + // comparison that passes because the expectations moved is not a passing comparison. + const baseline = await phaseBHashes(); + expect(Object.keys(baseline).sort()).toEqual([ + 'accept-corpus-freeze/accept-corpus-freeze.json', + 'frozen-expectations/frozen-expectation-set.json', + ]); + + for (const [artifact, phaseB] of Object.entries(baseline)) { + const parsed = JSON.parse(await readFile(join(LIVE_EVIDENCE, artifact), 'utf8')) as Record< + string, + unknown + >; + // Two comparisons, not one. The recorded field could have been edited to match + // edited bytes; the recomputation could not. + expect(parsed['contentHash']).toBe(phaseB); + expect(canonicalHash(parsed)).toBe(phaseB); + } + }); + + test('the drift check is green over the live trees after Phase E and Phase F', async () => { + const result = await checkFreezeHashes(LIVE_EVIDENCE); + expect(result.ok).toBe(true); + expect(result.findings).toEqual([]); + }); + + test('Phase F wrote its evidence outside the frozen trees', async () => { + // The comparison's own artifacts live in evidence/comparison/, which is not one of + // the two FREEZE_DIRS. If step (b) had written into a frozen tree, `checked` would + // name a third artifact. + const result = await checkFreezeHashes(LIVE_EVIDENCE); + expect(result.checked).toEqual([ + 'frozen-expectations/frozen-expectation-set.json', + 'accept-corpus-freeze/accept-corpus-freeze.json', + ]); + expect(await Bun.file(join(LIVE_EVIDENCE, 'comparison/diff-report.json')).exists()).toBe(true); + }); + + test('the T091 record agrees with what this test independently recomputed', async () => { + // The record is produced by the harness run; this test recomputes from the artifacts. + // Two derivations of the same fact, so a wrong record cannot pass unnoticed. + const record = (await Bun.file( + join(LIVE_EVIDENCE, 'comparison/expectations-unchanged.json'), + ).json()) as { + allUnchanged: boolean; + comparisons: readonly { + artifact: string; + phaseBRecordedHash: string; + recomputedNow: string; + unchangedSincePhaseB: boolean; + selfConsistentNow: boolean; + }[]; + }; + expect(record.allUnchanged).toBe(true); + + const baseline = await phaseBHashes(); + for (const entry of record.comparisons) { + expect(entry.unchangedSincePhaseB).toBe(true); + expect(entry.selfConsistentNow).toBe(true); + expect(entry.phaseBRecordedHash).toBe(baseline[entry.artifact] as string); + const parsed = JSON.parse( + await readFile(join(LIVE_EVIDENCE, entry.artifact), 'utf8'), + ) as Record; + expect(entry.recomputedNow).toBe(canonicalHash(parsed)); + } + }); + + test('observed failing: an amended expectation is caught against the Phase B baseline', async () => { + // The prohibition's failure mode, driven. An artifact edited to fit output and + // re-hashed is SELF-consistent — its own contentHash matches its own bytes — so the + // drift check alone would pass it. Comparing against Phase B's independently + // recorded value is what catches it, and that is why this assertion exists + // separately from T023's. + const original = JSON.parse( + await readFile(join(LIVE_EVIDENCE, 'frozen-expectations/frozen-expectation-set.json'), 'utf8'), + ) as Record; + const baseline = await phaseBHashes(); + + const { contentHash: _dropped, ...unsigned } = original; + const amended = { + ...unsigned, + derivedPathPatterns: [...(original['derivedPathPatterns'] as string[]), 'amended/**'], + }; + const resigned = { ...amended, contentHash: canonicalHash(amended) }; + + // Self-consistent, so the narrower drift check would not flag it... + expect(canonicalHash(resigned)).toBe(resigned.contentHash); + // ...and yet it is NOT the artifact Phase B recorded. + expect(resigned.contentHash).not.toBe( + baseline['frozen-expectations/frozen-expectation-set.json'], + ); + // The live artifact is untouched; nothing above was written to disk. + expect(original['contentHash']).toBe(baseline['frozen-expectations/frozen-expectation-set.json']); + }); +}); + describe('T023 — observed failing: a single mutated byte in a frozen artifact', () => { test('mutating one byte makes the drift check FAIL with the exact reason; restoring makes it PASS', async () => { const root = await resetTestDir(DIR_NAME); diff --git a/scripts/compare-accept-corpus.test.ts b/scripts/compare-accept-corpus.test.ts new file mode 100644 index 00000000..31759610 --- /dev/null +++ b/scripts/compare-accept-corpus.test.ts @@ -0,0 +1,645 @@ +/** + * T087 / T089 / T092 — checks on the clause-5 step (b) comparison harness. + * + * Three jobs: + * + * 1. **T087 — the harness exists, and its provenance is recorded.** R5 mechanism 3 is an + * *ordering* requirement: this harness had to be authored after the freeze and its + * audit. That cannot be asserted by a passing test, so what is checked here is that + * the record which carries it exists and says what it must. + * 2. **T089 — the gate has been observed failing.** The mutate → FAIL → restore → PASS + * cycle runs in-suite against the real corpus, and the retained negative case is + * checked for the verbatim strings the run emitted. + * 3. **T092 — no Phase F artifact claims correctness from a digest.** A populated, + * digest-verified envelope establishes integrity. This comparison establishes + * agreement with a maintainer-authored expectation set. Neither establishes that the + * adapter is correct, and every artifact this phase writes is scanned for that. + * + * The comparison kernel is pure, so every FAIL mode below is reachable without touching + * anything on disk — which is what lets the failure modes be *driven* rather than merely + * described. Nothing in this file writes to `evidence/frozen-expectations/` or + * `evidence/accept-corpus-freeze/`; `scripts/check-freeze-hashes.test.ts` re-asserts that + * independently (T091). + */ + +import { describe, expect, test } from 'bun:test'; +import { readFile, readdir } from 'node:fs/promises'; +import { join } from 'node:path'; +import { + type ExpectedEntity, + type ObservedEntity, + OBSERVE_FAILING_FLAG, + OverlayAbort, + REASON_ENTITY_MISSING, + REASON_FREEZE_DISAGREEMENT, + REASON_OWNERSHIP_STATE, + REASON_PATH_NOT_DERIVED, + REASON_PATH_NOT_EXPECTED, + REASON_PATH_ORDER, + REASON_PATTERN_UNION, + REASON_SOURCE_DOCUMENT, + REASON_UNEXPECTED_OWNERSHIP, + T089_MUTATION, + applyOverlay, + compare, + compareAcceptCorpus, + documentLineRanges, + readFrozenSide, + readOverlay, +} from './compare-accept-corpus.ts'; + +const REPO_ROOT = process.cwd(); +const EVIDENCE = join(REPO_ROOT, 'specs/010-catalog-backstage/evidence'); +const COMPARISON = join(EVIDENCE, 'comparison'); +const NEGATIVE_CASE = join(EVIDENCE, 'negative-cases/comparison-mismatch'); +const CORPUS = join(REPO_ROOT, 'specs/010-catalog-backstage/corpus'); + +// ── T087: the harness, its provenance, and the ordering it must respect ─────────────── + +describe('T087 — the comparison harness was authored after the freeze and its audit', () => { + test('the provenance record exists and names T024\u2019s confirmation', async () => { + const record = await readFile(join(COMPARISON, 'harness-provenance.md'), 'utf8'); + // R5 mechanism 3 cannot be proven by a passing assertion; the record is the only + // artifact that carries it, so what is checkable is that the record is present and + // cites the checkpoint that confirmed the absence. + expect(record).toContain('barrier-b-checkpoint.json'); + expect(record).toContain('mechanism3_ordering'); + expect(record).toContain('T024'); + }); + + test('the checkpoint it cites really does record that no harness existed', async () => { + const checkpoint = (await Bun.file(join(EVIDENCE, 'barrier-b-checkpoint.json')).json()) as { + BARRIER_B_CLEARED: boolean; + confirmations: { mechanism3_ordering: { cleared: boolean; claim: string } }; + }; + expect(checkpoint.BARRIER_B_CLEARED).toBe(true); + expect(checkpoint.confirmations.mechanism3_ordering.cleared).toBe(true); + expect(checkpoint.confirmations.mechanism3_ordering.claim).toContain( + 'No comparison harness exists anywhere in the repository', + ); + }); + + test('the two frozen artifacts still agree with each other, or the harness refuses to run', async () => { + // The freeze itself: "a divergence between them is a freeze failure, not a + // discrepancy to be reconciled". So the harness reads BOTH and cross-checks. + const frozen = readFrozenSide( + JSON.parse(await readFile(join(EVIDENCE, 'accept-corpus-freeze/accept-corpus-freeze.json'), 'utf8')), + JSON.parse(await readFile(join(EVIDENCE, 'frozen-expectations/frozen-expectation-set.json'), 'utf8')), + ); + expect(frozen.entities.length).toBe(24); + expect(frozen.derivedPathPatterns.length).toBe(25); + expect(frozen.repository).toBe('github.com/backstage/community-plugins'); + }); + + test('a disagreement between the two frozen artifacts is refused, not reconciled', () => { + const freeze = { + corpusRef: { repository: 'github.com/x/y', commit: 'a'.repeat(40) }, + contentHash: 'f'.repeat(64), + expectedPaths: [ + { + canonicalId: 'component:default/one', + ownershipState: 'explicit-paths', + sourcePath: 'a/catalog-info.yaml', + documentIndexInFile: 0, + expectedPaths: ['a/**'], + }, + ], + }; + const oracle = { + contentHash: 'e'.repeat(64), + derivedPathPatterns: ['a/**'], + expectedByEntity: [ + { canonicalId: 'component:default/one', ownershipState: 'explicit-paths', expectedPaths: ['b/**'] }, + ], + }; + expect(() => readFrozenSide(freeze, oracle)).toThrow(REASON_FREEZE_DISAGREEMENT); + // And the agreeing pair is accepted, so the check is not always-fail. + oracle.expectedByEntity[0]!.expectedPaths = ['a/**']; + expect(readFrozenSide(freeze, oracle).entities.length).toBe(1); + }); +}); + +// ── The overlay: applied in memory, never onto the vendored tree ────────────────────── + +describe('T088 — the overlay is applied at generation time, and the vendored tree is pristine', () => { + test('no vendored descriptor carries the annotation, before or after a full run', async () => { + // Ordered deliberately: read, run the whole comparison, read again. If the harness + // ever wrote an overlaid descriptor back, this is where it would show. + const files = (await readdir(CORPUS, { recursive: true })) + .map(String) + .filter((entry) => entry.endsWith('catalog-info.yaml')); + expect(files.length).toBe(24); + + const before = await Promise.all(files.map((file) => readFile(join(CORPUS, file), 'utf8'))); + expect(before.every((text) => !text.includes('adrkit.io/owned-paths'))).toBe(true); + + await compareAcceptCorpus(REPO_ROOT); + + const after = await Promise.all(files.map((file) => readFile(join(CORPUS, file), 'utf8'))); + expect(after).toEqual(before); + }); + + test('the overlay artifact carries 23 entries \u2014 one fewer than the corpus size', async () => { + // 24 selected entities, 23 overlay entries: the freeze exercises one + // `annotation-absent` entity, which by definition has no annotation to overlay. + // Read from accept-corpus-freeze.json's overlayProvenance.entries. + const overlay = readOverlay( + JSON.parse(await readFile(join(EVIDENCE, 'accept-corpus-freeze/overlay.json'), 'utf8')), + ); + expect(overlay.length).toBe(23); + expect(overlay.every((entry) => entry.documentIndexInFile === 0)).toBe(true); + }); + + test('injection into a descriptor with no annotations block adds exactly one annotation', () => { + const pristine = ['apiVersion: backstage.io/v1alpha1', 'kind: Component', 'metadata:', ' name: one', 'spec:', ' type: website', ''].join( + '\n', + ); + const overlaid = applyOverlay('a/catalog-info.yaml', pristine, 0, '["a/**"]'); + expect(overlaid).toContain(" adrkit.io/owned-paths: '[\"a/**\"]'"); + expect(overlaid).toContain(' annotations:'); + // Every pristine line survives, in order. + for (const line of pristine.split('\n')) expect(overlaid).toContain(line); + }); + + test('injection into a descriptor that already has annotations reuses the existing block', () => { + const pristine = [ + 'apiVersion: backstage.io/v1alpha1', + 'kind: Component', + 'metadata:', + ' name: one', + ' annotations:', + ' backstage.io/view-url: https://example.invalid', + 'spec:', + ' type: website', + '', + ].join('\n'); + const overlaid = applyOverlay('a/catalog-info.yaml', pristine, 0, '["a/**"]'); + expect(overlaid.match(/annotations:/g)?.length).toBe(1); + expect(overlaid).toContain(' backstage.io/view-url: https://example.invalid'); + }); + + test('a commented-out `# annotations:` is not mistaken for a real one', () => { + // Two corpus descriptors carry exactly this as a documentation example, and treating + // it as a mapping key would produce a descriptor whose annotation is inside a comment. + const pristine = [ + 'apiVersion: backstage.io/v1alpha1', + 'kind: Component', + 'metadata:', + ' name: one', + ' # Example for optional annotations', + ' # annotations:', + ' # github.com/project-slug: backstage/backstage', + 'spec:', + ' type: website', + '', + ].join('\n'); + const overlaid = applyOverlay('a/catalog-info.yaml', pristine, 0, '["a/**"]'); + expect(overlaid).toContain(' annotations:\n adrkit.io/owned-paths:'); + expect(overlaid).toContain(' # annotations:'); + }); + + test('the value is emitted as a string scalar, not a flow sequence', () => { + // `owned-paths-annotation.md` §1 step 2 requires a YAML *string* scalar. An unquoted + // `["a/**"]` parses as a sequence and is rejected at step 2 — a different case. + const pristine = 'apiVersion: backstage.io/v1alpha1\nkind: Component\nmetadata:\n name: one\n'; + const overlaid = applyOverlay('a/catalog-info.yaml', pristine, 0, '["a/**"]'); + expect(overlaid).toContain(`adrkit.io/owned-paths: '["a/**"]'`); + }); + + test('an overlay targeting a document the file does not hold aborts', () => { + const pristine = 'apiVersion: backstage.io/v1alpha1\nkind: Component\nmetadata:\n name: one\n'; + expect(() => applyOverlay('a/catalog-info.yaml', pristine, 3, '["a/**"]')).toThrow(OverlayAbort); + }); + + test('a document with no top-level `metadata:` aborts rather than being patched blindly', () => { + expect(() => applyOverlay('a/catalog-info.yaml', 'kind: Component\n', 0, '["a/**"]')).toThrow( + OverlayAbort, + ); + }); + + test('document ranges split a multi-document file the way the generator reads it', () => { + const lines = ['# c', 'kind: A', '---', 'kind: B', '']; + expect(documentLineRanges(lines)).toEqual([ + { start: 0, end: 2 }, + { start: 3, end: 5 }, + ]); + // A leading `---` does not create an empty document before it. + expect(documentLineRanges(['---', 'kind: A'])).toEqual([{ start: 1, end: 2 }]); + }); + + test('the real multi-document corpus file is overlaid on document 0 only', async () => { + const path = 'workspaces/acr/plugins/acr/catalog-info.yaml'; + const pristine = await readFile(join(CORPUS, path), 'utf8'); + const overlaid = applyOverlay(path, pristine, 0, '["a/**"]'); + const [first = '', second = ''] = overlaid.split('\n---\n'); + expect(first).toContain('adrkit.io/owned-paths'); + expect(second).not.toContain('adrkit.io/owned-paths'); + }); +}); + +// ── The comparison kernel: PASS on the corpus, and every FAIL mode driven ───────────── + +const BASE_EXPECTED: readonly ExpectedEntity[] = [ + { + canonicalId: 'component:default/one', + ownershipState: 'explicit-paths', + expectedPaths: ['a/**', 'b/**'], + sourcePath: 'one/catalog-info.yaml', + documentIndexInFile: 0, + }, +]; + +const BASE_OBSERVED: readonly ObservedEntity[] = [ + { + canonicalId: 'component:default/one', + ownershipState: 'explicit-paths', + derivedPaths: ['a/**', 'b/**'], + sourcePath: 'one/catalog-info.yaml', + documentIndexInFile: 0, + }, +]; + +const BASE_UNION = ['a/**', 'b/**']; + +describe('T088 — the comparison kernel discriminates, in both directions', () => { + test('the baseline passes, so every failure below is a real discrimination', () => { + const outcome = compare(BASE_EXPECTED, BASE_OBSERVED, BASE_UNION); + expect(outcome.pass).toBe(true); + expect(outcome.falsePositives).toEqual([]); + expect(outcome.falseNegatives).toEqual([]); + expect(outcome.otherMismatches).toEqual([]); + }); + + test('a missing entity is a false negative', () => { + const outcome = compare(BASE_EXPECTED, [], []); + expect(outcome.pass).toBe(false); + expect(outcome.falseNegatives[0]?.reason).toBe(REASON_ENTITY_MISSING); + }); + + test('an expected path that was not derived is a false negative', () => { + const observed = [{ ...BASE_OBSERVED[0]!, derivedPaths: ['a/**'] }]; + const outcome = compare(BASE_EXPECTED, observed, ['a/**']); + expect(outcome.pass).toBe(false); + expect(outcome.falseNegatives.map((f) => f.reason)).toContain(REASON_PATH_NOT_DERIVED); + }); + + test('a derived path that was not expected is a false positive', () => { + const observed = [{ ...BASE_OBSERVED[0]!, derivedPaths: ['a/**', 'b/**', 'c/**'] }]; + const outcome = compare(BASE_EXPECTED, observed, ['a/**', 'b/**', 'c/**']); + expect(outcome.pass).toBe(false); + expect(outcome.falsePositives.map((f) => f.reason)).toContain(REASON_PATH_NOT_EXPECTED); + }); + + test('a wrong ownershipState is a mismatch, and is not reported as a path defect', () => { + // `explicit-empty` and `annotation-absent` both yield an empty array; + // `owned-paths-annotation.md` §3 forbids inferring the distinction from the paths. + const expected = [{ ...BASE_EXPECTED[0]!, ownershipState: 'explicit-empty', expectedPaths: [] }]; + const observed = [{ ...BASE_OBSERVED[0]!, ownershipState: 'annotation-absent', derivedPaths: [] }]; + const outcome = compare(expected, observed, []); + expect(outcome.pass).toBe(false); + expect(outcome.otherMismatches.map((f) => f.reason)).toContain(REASON_OWNERSHIP_STATE); + expect(outcome.falsePositives).toEqual([]); + expect(outcome.falseNegatives).toEqual([]); + }); + + test('the right members in the wrong order is a mismatch', () => { + // The whole ADR-0020 clause-6 re-freeze exists because the spike's oracle recorded + // input order. Same members, wrong order, must not pass. + const observed = [{ ...BASE_OBSERVED[0]!, derivedPaths: ['b/**', 'a/**'] }]; + const outcome = compare(BASE_EXPECTED, observed, BASE_UNION); + expect(outcome.pass).toBe(false); + expect(outcome.otherMismatches.map((f) => f.reason)).toContain(REASON_PATH_ORDER); + expect(outcome.falsePositives).toEqual([]); + expect(outcome.falseNegatives).toEqual([]); + }); + + test('a different source document for the same canonical id is a mismatch', () => { + const observed = [{ ...BASE_OBSERVED[0]!, documentIndexInFile: 1 }]; + const outcome = compare(BASE_EXPECTED, observed, BASE_UNION); + expect(outcome.pass).toBe(false); + expect(outcome.otherMismatches.map((f) => f.reason)).toContain(REASON_SOURCE_DOCUMENT); + }); + + test('a pattern union that disagrees with the oracle is a mismatch', () => { + const outcome = compare(BASE_EXPECTED, BASE_OBSERVED, ['a/**', 'b/**', 'c/**']); + expect(outcome.pass).toBe(false); + expect(outcome.otherMismatches.map((f) => f.reason)).toContain(REASON_PATTERN_UNION); + }); + + test('an entity outside the expectation set that derives NOTHING is not a finding', () => { + // The descriptor-file / entity-document counting trap. A manifest names files; the + // envelope carries one entity per admissible document in them. An unselected sibling + // carries no overlay, derives nothing, and asserts no ownership. + const sibling: ObservedEntity = { + canonicalId: 'component:default/sibling', + ownershipState: 'annotation-absent', + derivedPaths: [], + sourcePath: 'one/catalog-info.yaml', + documentIndexInFile: 1, + }; + const outcome = compare(BASE_EXPECTED, [...BASE_OBSERVED, sibling], BASE_UNION); + expect(outcome.pass).toBe(true); + expect(outcome.entitiesOutsideTheExpectationSet.map((e) => e.canonicalId)).toEqual([ + 'component:default/sibling', + ]); + }); + + test('an entity outside the expectation set that DOES derive paths is a false positive', () => { + const sibling: ObservedEntity = { + canonicalId: 'component:default/sibling', + ownershipState: 'explicit-paths', + derivedPaths: ['z/**'], + sourcePath: 'one/catalog-info.yaml', + documentIndexInFile: 1, + }; + const outcome = compare(BASE_EXPECTED, [...BASE_OBSERVED, sibling], BASE_UNION); + expect(outcome.pass).toBe(false); + expect(outcome.falsePositives.map((f) => f.reason)).toContain(REASON_UNEXPECTED_OWNERSHIP); + }); +}); + +// ── T088: the real run, and the report it wrote ─────────────────────────────────────── + +describe('T088 / SC-011 \u2014 the real comparison over the frozen accept corpus', () => { + test('zero false positives and zero false negatives over all 24 annotated entities', async () => { + const report = await compareAcceptCorpus(REPO_ROOT); + expect(report.verdict).toBe('PASS'); + expect(report.outcome.falsePositives).toEqual([]); + expect(report.outcome.falseNegatives).toEqual([]); + expect(report.outcome.otherMismatches).toEqual([]); + expect(report.frozen.entities.length).toBe(24); + }); + + test('the envelope is populated, and carries MORE entities than the corpus has files', async () => { + const report = await compareAcceptCorpus(REPO_ROOT); + expect(report.run.envelope.entities.length).toBeGreaterThan(24); + expect(report.run.envelope.entities.some((entity) => entity.derivedPaths.length > 0)).toBe(true); + // Every entity, selected or not, is a maintainer overlay: no third-party descriptor + // in the pinned corpus carries the annotation (accept-corpus-freeze overlayProvenance). + expect(report.run.envelope.entities.every((entity) => entity.provenance === 'maintainer-overlay')).toBe( + true, + ); + }); + + test('all three ownership states are exercised, in the proportions the freeze records', async () => { + const report = await compareAcceptCorpus(REPO_ROOT); + const byId = new Map(report.run.observed.map((entity) => [entity.canonicalId, entity] as const)); + const states = report.frozen.entities.map((entity) => byId.get(entity.canonicalId)?.ownershipState); + expect(states.filter((state) => state === 'explicit-paths').length).toBe(22); + expect(states.filter((state) => state === 'explicit-empty').length).toBe(1); + expect(states.filter((state) => state === 'annotation-absent').length).toBe(1); + }); + + test('the run is deterministic \u2014 two comparisons agree exactly', async () => { + const [first, second] = await Promise.all([ + compareAcceptCorpus(REPO_ROOT), + compareAcceptCorpus(REPO_ROOT), + ]); + expect(JSON.stringify(second.run.envelope)).toBe(JSON.stringify(first.run.envelope)); + expect(second.run.overlaidDigests).toEqual(first.run.overlaidDigests); + }); + + test('the committed diff report records the PASS and the counts it was run at', async () => { + const report = (await Bun.file(join(COMPARISON, 'diff-report.json')).json()) as { + verdict: string; + counts: Record; + findings: Record; + }; + expect(report.verdict).toBe('PASS'); + expect(report.counts['expectedEntities']).toBe(24); + expect(report.counts['falsePositives']).toBe(0); + expect(report.counts['falseNegatives']).toBe(0); + expect(report.counts['otherMismatches']).toBe(0); + expect(report.findings['falsePositives']).toEqual([]); + expect(report.findings['falseNegatives']).toEqual([]); + }); +}); + +// ── T089: observed failing ──────────────────────────────────────────────────────────── + +describe('T089 \u2014 observed failing: a mutated comparison input fails the gate', () => { + test('mutate \u2192 FAIL with one false positive and one false negative \u2192 restore \u2192 PASS', async () => { + const mutated = await compareAcceptCorpus(REPO_ROOT, T089_MUTATION); + expect(mutated.verdict).toBe('FAIL'); + expect(mutated.outcome.falseNegatives.length).toBe(1); + expect(mutated.outcome.falsePositives.length).toBe(1); + expect(mutated.outcome.falseNegatives[0]?.reason).toBe(REASON_PATH_NOT_DERIVED); + expect(mutated.outcome.falsePositives[0]?.reason).toBe(REASON_PATH_NOT_EXPECTED); + expect(mutated.outcome.falseNegatives[0]?.canonicalId).toBe( + 'component:default/backstage-plugin-adr-backend', + ); + // And the corpus-wide union moves with it: 26 patterns where the oracle records 25. + expect(mutated.outcome.otherMismatches.map((f) => f.reason)).toContain(REASON_PATTERN_UNION); + + const restored = await compareAcceptCorpus(REPO_ROOT); + expect(restored.verdict).toBe('PASS'); + }); + + test('the mutation touches the corpus side only \u2014 the frozen expectations are identical', async () => { + // The prohibition, made checkable: the same frozen bytes are read on both runs. + const before = await readFile( + join(EVIDENCE, 'frozen-expectations/frozen-expectation-set.json'), + 'utf8', + ); + await compareAcceptCorpus(REPO_ROOT, T089_MUTATION); + const after = await readFile( + join(EVIDENCE, 'frozen-expectations/frozen-expectation-set.json'), + 'utf8', + ); + expect(after).toBe(before); + }); + + test('the retained negative case holds the verbatim FAIL output and the restored PASS', async () => { + const failed = await readFile(join(NEGATIVE_CASE, 'observed-fail.txt'), 'utf8'); + expect(failed).toContain('compare-accept-corpus: FAIL'); + expect(failed).toContain('1 false positive(s), 1 false negative(s)'); + expect(failed).toContain(REASON_PATH_NOT_DERIVED); + expect(failed).toContain(REASON_PATH_NOT_EXPECTED); + expect(failed).toContain('exit=1'); + + const restored = await readFile(join(NEGATIVE_CASE, 'restored.observed.txt'), 'utf8'); + expect(restored).toContain('compare-accept-corpus: PASS'); + expect(restored).toContain('0 false positive(s), 0 false negative(s)'); + expect(restored).toContain('exit=0'); + }); + + test('the negative case README names the command that produced the failure', async () => { + // negative-cases/README.md: "Always record which command produced the failure." + const readme = await readFile(join(NEGATIVE_CASE, 'README.md'), 'utf8'); + expect(readme).toContain(OBSERVE_FAILING_FLAG); + expect(readme).toContain('scripts/compare-accept-corpus.ts'); + expect(readme).toContain(T089_MUTATION.sourcePath); + }); + + test('the observe-failing flag is off by default, so the committed report is unmutated', async () => { + const report = await compareAcceptCorpus(REPO_ROOT); + expect(report.verdict).toBe('PASS'); + const committed = (await Bun.file(join(COMPARISON, 'diff-report.json')).json()) as { + verdict: string; + }; + expect(committed.verdict).toBe('PASS'); + }); +}); + +// ── T090: step (b)'s own record ─────────────────────────────────────────────────────── + +describe('T090 \u2014 step (b) records its own hashes and its own verdict', () => { + test('the record recomputes the frozen hashes rather than transcribing them', async () => { + const record = (await Bun.file(join(COMPARISON, 'step-b-record.json')).json()) as { + verdict: string; + inheritsFromStepA: boolean; + recomputedFrozenHashes: Record; + ownHashes: Record; + }; + expect(record.verdict).toBe('PASS'); + expect(record.inheritsFromStepA).toBe(false); + for (const entry of Object.values(record.recomputedFrozenHashes)) { + expect(entry.match).toBe(true); + expect(entry.recomputed).toBe(entry.recorded); + } + expect(Object.keys(record.ownHashes).length).toBeGreaterThan(0); + }); + + test('step (b)\u2019s own hashes cover the corpus, the overlaid inputs and the envelope', async () => { + const record = (await Bun.file(join(COMPARISON, 'step-b-record.json')).json()) as { + ownHashes: { + envelopeDigest: string; + overlaidSourceDigests: Record; + vendoredCorpusSha256: Record; + }; + }; + expect(record.ownHashes.envelopeDigest).toMatch(/^[0-9a-f]{64}$/); + expect(Object.keys(record.ownHashes.overlaidSourceDigests).length).toBe(24); + expect(Object.keys(record.ownHashes.vendoredCorpusSha256).length).toBe(24); + }); + + test('the recorded envelope digest is the digest a fresh run actually produces', async () => { + const record = (await Bun.file(join(COMPARISON, 'step-b-record.json')).json()) as { + ownHashes: { envelopeDigest: string; overlaidSourceDigests: Record }; + }; + const report = await compareAcceptCorpus(REPO_ROOT); + expect(report.run.envelope.digest).toBe(record.ownHashes.envelopeDigest); + expect(report.run.overlaidDigests).toEqual(record.ownHashes.overlaidSourceDigests); + }); +}); + +// ── T092: reporting honesty ─────────────────────────────────────────────────────────── + +describe('T092 \u2014 no Phase F artifact presents a digest as evidence of correctness', () => { + /** + * Every artifact this phase writes, plus the two scripts that write them. + * + * The scan reads whole files, prose included, because the claim being guarded is a + * claim in prose. This test file is excluded for the reason `test/source-scan.ts` + * names: it must contain the forbidden phrases in order to forbid them. + */ + const SCANNED = [ + 'specs/010-catalog-backstage/evidence/comparison/harness-provenance.md', + 'specs/010-catalog-backstage/evidence/comparison/reporting-honesty.md', + 'specs/010-catalog-backstage/evidence/comparison/diff-report.json', + 'specs/010-catalog-backstage/evidence/comparison/step-b-record.json', + 'specs/010-catalog-backstage/evidence/comparison/expectations-unchanged.json', + 'specs/010-catalog-backstage/evidence/negative-cases/comparison-mismatch/README.md', + 'scripts/compare-accept-corpus.ts', + 'scripts/vendor-accept-corpus.ts', + 'specs/010-catalog-backstage/corpus/README.md', + 'specs/010-catalog-backstage/corpus/VENDOR-MANIFEST.json', + ] as const; + + /** + * Affirmative overclaims. Each is chosen so it cannot occur inside a correct + * disclaimer — "does not establish correctness" contains neither "establishes + * correctness" nor "proves correctness", so a truthful artifact never trips these. + */ + const FORBIDDEN = [ + 'proves correctness', + 'proves the derived ownership is correct', + 'establishes correctness', + 'demonstrates correctness', + 'proof of correctness', + 'evidence of correctness', + 'is reference-verified', + 'is externally validated', + 'ready for release', + 'ready to release', + 'rung 2 evidence', + 'rung 3 evidence', + ] as const; + + test('every scanned artifact exists, so an empty finding set is not an empty scan', async () => { + for (const path of SCANNED) { + expect(await Bun.file(join(REPO_ROOT, path)).exists()).toBe(true); + } + }); + + test('no scanned artifact contains an affirmative correctness or rung overclaim', async () => { + const offences: string[] = []; + for (const path of SCANNED) { + const text = (await readFile(join(REPO_ROOT, path), 'utf8')).toLowerCase(); + for (const phrase of FORBIDDEN) { + if (text.includes(phrase)) offences.push(`${path}: ${phrase}`); + } + } + expect(offences).toEqual([]); + }); + + test('the scan can see an offence, so a clean result means it looked', () => { + const fixture = 'The populated envelope proves correctness of the derived ownership.'; + expect(FORBIDDEN.some((phrase) => fixture.toLowerCase().includes(phrase))).toBe(true); + const honest = 'A digest establishes integrity; it does not establish correctness.'; + expect(FORBIDDEN.some((phrase) => honest.toLowerCase().includes(phrase))).toBe(false); + }); + + test('the two documents that carry the framing say integrity is not correctness', async () => { + for (const path of [ + 'specs/010-catalog-backstage/evidence/comparison/reporting-honesty.md', + 'specs/010-catalog-backstage/evidence/comparison/harness-provenance.md', + ]) { + const text = await readFile(join(REPO_ROOT, path), 'utf8'); + expect(text.toLowerCase()).toContain('integrity'); + expect(text.toLowerCase()).toContain('correctness'); + expect(text).toContain('rung 1'); + } + }); + + test('the diff report scopes what it establishes and lists what it does not', async () => { + const report = (await Bun.file(join(COMPARISON, 'diff-report.json')).json()) as { + whatThisDoesAndDoesNotEstablish: { establishes: string; doesNotEstablish: string[] }; + }; + const section = report.whatThisDoesAndDoesNotEstablish; + expect(section.establishes).toContain('maintainer-authored expectation set'); + expect(section.doesNotEstablish.length).toBeGreaterThanOrEqual(4); + expect(section.doesNotEstablish.join(' ')).toContain('INTEGRITY, not correctness'); + expect(section.doesNotEstablish.join(' ')).toContain('rung 1 only'); + }); + + test('no Phase F artifact describes maintainer verification as external or third-party', async () => { + // ADR-0014's honesty rule: only the corpus DATA is third-party, never the validation. + // The phrases below are affirmative constructions only. A negated disclaimer — "which + // is not external, third-party, or community validation" — is the correct thing to + // write and must not trip this, which is the mistake the first draft of this check + // made and which `negative-cases/README.md` makes verbatim. + const AFFIRMATIVE_ADOPTION_CLAIMS = [ + 'externally validated by', + 'third-party validation of', + 'validated by the community', + 'community-validated', + 'was independently validated', + ] as const; + + const offences: string[] = []; + for (const path of SCANNED) { + const text = (await readFile(join(REPO_ROOT, path), 'utf8')).toLowerCase(); + for (const phrase of AFFIRMATIVE_ADOPTION_CLAIMS) { + if (text.includes(phrase)) offences.push(`${path}: ${phrase}`); + } + } + expect(offences).toEqual([]); + + // Driven, so a clean result means the scan looked. + const overclaim = 'the adapter was externally validated by two downstream teams.'; + expect(AFFIRMATIVE_ADOPTION_CLAIMS.some((phrase) => overclaim.includes(phrase))).toBe(true); + const disclaimer = 'this observation is maintainer-owned, which is not community validation.'; + expect(AFFIRMATIVE_ADOPTION_CLAIMS.some((phrase) => disclaimer.includes(phrase))).toBe(false); + }); +}); diff --git a/scripts/compare-accept-corpus.ts b/scripts/compare-accept-corpus.ts new file mode 100644 index 00000000..34752f6d --- /dev/null +++ b/scripts/compare-accept-corpus.ts @@ -0,0 +1,1195 @@ +/** + * T087 / T088 / T092 — ADR-0020 clause 5 **step (b)**: the post-output comparison. + * + * # What step (b) is for, and why it is a separate step + * + * Step (a) — the pre-output freeze and its independent audit — is Barrier B, and it is + * done. It fixed, in advance, what ownership derivation over the accept corpus is + * *expected* to produce. It did not show that anything produces it. + * + * ADR-0020 clause 5 is explicit about why that is not enough: + * + * > Freezing the expected paths is not enough; the output must be compared against + * > them. A populated, digest-verified envelope proves integrity, not correctness — a + * > semantically wrong envelope can carry a perfectly valid self-digest. + * + * So this harness runs the assembled generator over the frozen accept corpus and diffs + * the ownership it derived against the frozen expectations, requiring **zero false + * positives and zero false negatives** (FR-056, SC-011). + * + * # The one rule that governs everything here + * + * **The expectations are never amended to fit the output.** If the two disagree, either + * the output is wrong or the expectation is wrong, and the answer is to report it. A + * comparison that passes because the expectations moved is not a passing comparison — + * `scripts/check-freeze-hashes.ts` fails the build on any drift in the frozen trees, and + * `check-freeze-hashes.test.ts` re-asserts the Phase B hashes across the whole of E and F + * (T091). A mismatch is a finding, not an obstacle. + * + * # R5 mechanism 3 — this file was authored *after* the freeze and its audit + * + * `plan.md`'s third enforcement mechanism is an ordering requirement: the harness that + * reads both generator output *and* the frozen expectations is written after the freeze, + * never before, because a harness authored first collapses clause 5's two distinct steps + * into one and step (b)'s PASS then inherits from step (a) instead of standing alone. + * + * T024's checkpoint (`evidence/barrier-b-checkpoint.json`, `mechanism3_ordering`) records + * that no comparison harness existed anywhere in the repository at that point. The + * provenance record at `evidence/comparison/harness-provenance.md` carries the rest. + * + * # What the corpus is, and where the overlay lives + * + * `specs/010-catalog-backstage/corpus/` holds the 24 selected descriptors as **pristine + * upstream bytes**, vendored by `scripts/vendor-accept-corpus.ts` (T086a) and verified + * against the content address the pinned commit fixes. The maintainer-authored + * `adrkit.io/owned-paths` overlay is **not** in those files. It stays in + * `evidence/accept-corpus-freeze/overlay.json` and is applied here, at generation time, + * into a temporary run directory that is deleted afterwards. + * + * The vendored tree is never written to. That is what keeps ADR-0020 clause 5's + * "authored upstream and otherwise unmodified" provable by digest, and what keeps + * `data-model.md` §10's upstream/maintainer boundary legible by inspection rather than by + * assertion. + * + * # Integrity is not correctness (FR-058, SC-012), and this harness does not blur it + * + * The envelope this harness generates carries a self-digest. That digest establishes + * **integrity** — the bytes were not accidentally corrupted or naively mutated. It + * establishes nothing about whether the derived ownership is semantically right. + * + * What this comparison establishes is narrower than "the adapter is correct": it + * establishes **agreement with a maintainer-authored expectation set** over one frozen + * corpus. The expectations were written by hand from frozen contracts, by the maintainer. + * Agreement between our generator and our expectations is evidence that the + * implementation matches the specification we wrote; it is not independent evidence that + * either is right, and it says nothing about Backstage as a running system. + * + * `evidence/comparison/reporting-honesty.md` states this at length, and + * `scripts/compare-accept-corpus.test.ts` asserts that no Phase F artifact says otherwise. + * + * @see `docs/adr/0020-rescope-sc-010-and-authorize-work-toward-the-backstage-catalog-adapter.md` clause 5 + * @see `specs/010-catalog-backstage/spec.md` FR-056, FR-057, FR-058, SC-011, SC-012 + * @see `specs/010-catalog-backstage/plan.md` — Barrier B, R5 mechanism 3 + */ + +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +// `compareCodeUnits` is imported by path rather than through `@adrkit/core`'s package +// name. Bun's isolated linker installs workspace dependencies under the *depending* +// package, so `@adrkit/core` is not resolvable from `scripts/`; and this is the exact +// module the freeze cites as the definition of its ordering +// (`accept-corpus-freeze/expected-paths.json`: "compareCodeUnits — whose entire +// definition is `a < b ? -1 : a > b ? 1 : 0` … packages/core/src/ordering/index.ts"). +// One comparator, named at its source, rather than a second one written here. +import { compareCodeUnits } from '../packages/core/src/ordering/index.ts'; +import { canonicalHash } from './audit-oracle-freeze.ts'; +import { sha256Hex } from '../packages/adapters/catalog-backstage/src/manifest/digests.ts'; +import { readDescriptorDocuments } from '../packages/adapters/catalog-backstage/src/descriptor/read.ts'; +import { allMaintainerOverlay } from '../packages/adapters/catalog-backstage/src/envelope/provenance.ts'; +import type { SnapshotEnvelope } from '../packages/adapters/catalog-backstage/src/envelope/shape.ts'; +import { runGeneration } from '../packages/adapters/catalog-backstage/src/pipeline.ts'; + +/** The annotation key the overlay writes. `owned-paths-annotation.md` §1. */ +export const OWNED_PATHS_ANNOTATION = 'adrkit.io/owned-paths'; + +/** Repository-relative paths this harness reads. None of them is ever written. */ +export const CORPUS_DIR = 'specs/010-catalog-backstage/corpus'; +export const EVIDENCE_DIR = 'specs/010-catalog-backstage/evidence'; +export const FREEZE_PATH = `${EVIDENCE_DIR}/accept-corpus-freeze/accept-corpus-freeze.json`; +export const OVERLAY_PATH = `${EVIDENCE_DIR}/accept-corpus-freeze/overlay.json`; +export const ORACLE_PATH = `${EVIDENCE_DIR}/frozen-expectations/frozen-expectation-set.json`; + +/** Where the diff report is written. */ +export const DIFF_REPORT_PATH = `${EVIDENCE_DIR}/comparison/diff-report.json`; + +/** Where step (b)'s own record and the prohibition guard's record are written. */ +export const STEP_B_RECORD_PATH = `${EVIDENCE_DIR}/comparison/step-b-record.json`; +export const EXPECTATIONS_UNCHANGED_PATH = `${EVIDENCE_DIR}/comparison/expectations-unchanged.json`; + +/** The Barrier B checkpoint, which is where Phase B's hash values were independently recorded. */ +export const CHECKPOINT_PATH = `${EVIDENCE_DIR}/barrier-b-checkpoint.json`; + +/** The two frozen artifacts, relative to the evidence directory. */ +export const FROZEN_ARTIFACTS = [ + 'frozen-expectations/frozen-expectation-set.json', + 'accept-corpus-freeze/accept-corpus-freeze.json', +] as const; + +// ── Finding kinds ───────────────────────────────────────────────────────────────────── + +/** + * The three ways output and expectation can disagree. + * + * FR-056 fixes two of them by name and then says "**any** mismatch fails the gate", so + * the third is carried explicitly rather than folded into one of the first two. Folding a + * wrong `ownershipState` into "false positive" would report a path defect that did not + * happen; dropping it would let a real mismatch through. + */ +export type FindingKind = 'false-positive' | 'false-negative' | 'other-mismatch'; + +export interface Finding { + readonly kind: FindingKind; + /** A stable, greppable reason. Asserted verbatim by the tests and the negative case. */ + readonly reason: string; + readonly canonicalId: string; + readonly detail: string; +} + +/** Reasons, as constants, so a test and an evidence file cannot drift from the code. */ +export const REASON_ENTITY_MISSING = 'expected entity absent from generator output'; +export const REASON_PATH_NOT_DERIVED = 'expected path was not derived'; +export const REASON_PATH_NOT_EXPECTED = 'derived path is not in the frozen expectation'; +export const REASON_UNEXPECTED_OWNERSHIP = + 'entity outside the frozen expectation set derived a non-empty path set'; +export const REASON_OWNERSHIP_STATE = 'ownershipState does not match the frozen expectation'; +export const REASON_PATH_ORDER = + 'derivedPaths carry the expected members in a different order than the frozen expectation'; +export const REASON_SOURCE_DOCUMENT = + 'sourceDocument does not match the (sourcePath, documentIndexInFile) the freeze records'; +export const REASON_PATTERN_UNION = + 'the union of derived patterns does not match the oracle derivedPathPatterns'; +export const REASON_FREEZE_DISAGREEMENT = + 'accept-corpus-freeze and frozen-expectation-set disagree, which is a freeze failure rather than a comparison finding'; + +// ── The frozen side ─────────────────────────────────────────────────────────────────── + +/** One frozen expectation, assembled from both frozen artifacts. */ +export interface ExpectedEntity { + readonly canonicalId: string; + readonly ownershipState: string; + readonly expectedPaths: readonly string[]; + readonly sourcePath: string; + readonly documentIndexInFile: number; +} + +/** One entity as the generator actually emitted it. */ +export interface ObservedEntity { + readonly canonicalId: string; + readonly ownershipState: string; + readonly derivedPaths: readonly string[]; + readonly sourcePath: string; + readonly documentIndexInFile: number; +} + +export interface FrozenSide { + readonly repository: string; + readonly commit: string; + readonly entities: readonly ExpectedEntity[]; + /** The oracle's 25-element deduplicated pattern union, in `compareCodeUnits` order. */ + readonly derivedPathPatterns: readonly string[]; + readonly freezeContentHash: string; + readonly oracleContentHash: string; +} + +function asRecord(value: unknown, what: string): Record { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new Error(`${what} is not a JSON object`); + } + return value as Record; +} + +function asStringArray(value: unknown, what: string): readonly string[] { + if (!Array.isArray(value) || value.some((one) => typeof one !== 'string')) { + throw new Error(`${what} is not an array of strings`); + } + return value as readonly string[]; +} + +/** + * Assemble the frozen side from **both** frozen artifacts, and cross-check them. + * + * `accept-corpus-freeze.json` records `(canonicalId, ownershipState, sourcePath, + * documentIndexInFile, expectedPaths)`; `frozen-expectation-set.json` records + * `(canonicalId, ownershipState, expectedPaths)` plus the pattern union. The freeze + * itself says the two "are two views of one freeze, and a divergence between them is a + * freeze failure, not a discrepancy to be reconciled" — so this reads both and refuses to + * proceed if they disagree, rather than silently preferring one. + */ +export function readFrozenSide(freezeJson: unknown, oracleJson: unknown): FrozenSide { + const freeze = asRecord(freezeJson, 'accept-corpus-freeze.json'); + const oracle = asRecord(oracleJson, 'frozen-expectation-set.json'); + + const corpusRef = asRecord(freeze['corpusRef'], 'corpusRef'); + const repository = corpusRef['repository']; + const commit = corpusRef['commit']; + if (typeof repository !== 'string' || typeof commit !== 'string') { + throw new Error('corpusRef.repository / corpusRef.commit are missing or not strings'); + } + + const freezeEntries = freeze['expectedPaths']; + if (!Array.isArray(freezeEntries)) throw new Error('freeze expectedPaths is not an array'); + const oracleEntries = oracle['expectedByEntity']; + if (!Array.isArray(oracleEntries)) throw new Error('oracle expectedByEntity is not an array'); + + if (freezeEntries.length !== oracleEntries.length) { + throw new Error( + `${REASON_FREEZE_DISAGREEMENT}: freeze carries ${freezeEntries.length} entries, oracle carries ${oracleEntries.length}`, + ); + } + + const entities: ExpectedEntity[] = []; + for (const [index, raw] of freezeEntries.entries()) { + const entry = asRecord(raw, `freeze expectedPaths[${index}]`); + const mirror = asRecord(oracleEntries[index], `oracle expectedByEntity[${index}]`); + + const canonicalId = entry['canonicalId']; + const ownershipState = entry['ownershipState']; + const sourcePath = entry['sourcePath']; + const documentIndexInFile = entry['documentIndexInFile']; + if ( + typeof canonicalId !== 'string' || + typeof ownershipState !== 'string' || + typeof sourcePath !== 'string' || + typeof documentIndexInFile !== 'number' + ) { + throw new Error(`freeze expectedPaths[${index}] is malformed`); + } + const expectedPaths = asStringArray(entry['expectedPaths'], `freeze expectedPaths[${index}]`); + + // The cross-check. Field-by-field, in the order the two artifacts share. + if (mirror['canonicalId'] !== canonicalId) { + throw new Error( + `${REASON_FREEZE_DISAGREEMENT}: entry ${index} is ${String(canonicalId)} in the freeze and ${String(mirror['canonicalId'])} in the oracle`, + ); + } + if (mirror['ownershipState'] !== ownershipState) { + throw new Error( + `${REASON_FREEZE_DISAGREEMENT}: ${canonicalId} ownershipState is ${ownershipState} in the freeze and ${String(mirror['ownershipState'])} in the oracle`, + ); + } + const mirrorPaths = asStringArray(mirror['expectedPaths'], `oracle expectedByEntity[${index}]`); + if (mirrorPaths.length !== expectedPaths.length || mirrorPaths.some((p, i) => p !== expectedPaths[i])) { + throw new Error( + `${REASON_FREEZE_DISAGREEMENT}: ${canonicalId} expectedPaths differ between the two frozen artifacts`, + ); + } + + entities.push({ canonicalId, ownershipState, expectedPaths, sourcePath, documentIndexInFile }); + } + + const freezeContentHash = freeze['contentHash']; + const oracleContentHash = oracle['contentHash']; + if (typeof freezeContentHash !== 'string' || typeof oracleContentHash !== 'string') { + throw new Error('a frozen artifact carries no contentHash'); + } + + return { + repository, + commit, + entities, + derivedPathPatterns: asStringArray(oracle['derivedPathPatterns'], 'oracle derivedPathPatterns'), + freezeContentHash, + oracleContentHash, + }; +} + +/** One `(sourcePath, documentIndexInFile) -> annotationValue` overlay entry. */ +export interface OverlayEntry { + readonly sourcePath: string; + readonly documentIndexInFile: number; + readonly annotationValue: string; +} + +export function readOverlay(overlayJson: unknown): readonly OverlayEntry[] { + const overlay = asRecord(overlayJson, 'overlay.json'); + const entries = overlay['overlay']; + if (!Array.isArray(entries)) throw new Error('overlay.overlay is not an array'); + return entries.map((raw, index) => { + const entry = asRecord(raw, `overlay[${index}]`); + const sourcePath = entry['sourcePath']; + const documentIndexInFile = entry['documentIndexInFile']; + const annotationValue = entry['annotationValue']; + if ( + typeof sourcePath !== 'string' || + typeof documentIndexInFile !== 'number' || + typeof annotationValue !== 'string' + ) { + throw new Error(`overlay[${index}] is malformed`); + } + return { sourcePath, documentIndexInFile, annotationValue }; + }); +} + +// ── Overlay application (in memory, never onto the vendored tree) ───────────────────── + +/** Raised when a descriptor's shape defeats textual overlay application. */ +export class OverlayAbort extends Error { + constructor(message: string) { + super(message); + this.name = 'OverlayAbort'; + } +} + +/** + * Line ranges of each YAML document in `text`, in document order. + * + * A `---` at column 0 starts a document; a leading one does not create an empty document + * before it. This is deliberately simple, and it is not trusted: {@link applyOverlay} + * re-parses its own output with the generator's own reader and refuses to return text + * whose decoded documents differ from the pristine ones by anything other than the single + * injected annotation. A mis-split therefore fails loudly rather than producing a + * plausible-looking corpus. + */ +export function documentLineRanges(lines: readonly string[]): readonly { start: number; end: number }[] { + const boundaries: number[] = []; + for (const [index, line] of lines.entries()) { + if (/^---(\s|$)/.test(line)) boundaries.push(index); + } + + const ranges: { start: number; end: number }[] = []; + let start = 0; + for (const boundary of boundaries) { + if (boundary === 0) { + start = 1; + continue; + } + ranges.push({ start, end: boundary }); + start = boundary + 1; + } + ranges.push({ start, end: lines.length }); + return ranges; +} + +function leadingSpaces(line: string): number { + return line.length - line.trimStart().length; +} + +/** + * Inject one `adrkit.io/owned-paths` annotation into one document of `text`. + * + * Textual rather than a YAML round-trip, and for a reason: re-emitting the document would + * rewrite formatting, comments and scalar styles throughout, and clause 5's whole purpose + * is to exercise derivation against **real descriptor structure and real field shapes**. A + * re-serialized descriptor is no longer the shape upstream published. So every byte + * outside the inserted lines is carried through untouched. + * + * The value is emitted as a **single-quoted scalar**. `owned-paths-annotation.md` §1 step + * 2 requires a YAML *string* scalar, and an unquoted `["a/**"]` would parse as a flow + * sequence and be rejected at step 2 — a different case from the one the overlay means. + */ +export function applyOverlay( + sourcePath: string, + text: string, + documentIndexInFile: number, + annotationValue: string, +): string { + const lines = text.split('\n'); + const ranges = documentLineRanges(lines); + const range = ranges[documentIndexInFile]; + if (range === undefined) { + throw new OverlayAbort( + `${sourcePath}: overlay targets document ${documentIndexInFile}, but the file holds ${ranges.length}`, + ); + } + + let metadataLine = -1; + for (let index = range.start; index < range.end; index += 1) { + if (/^metadata:\s*$/.test(lines[index] as string)) { + metadataLine = index; + break; + } + } + if (metadataLine === -1) { + throw new OverlayAbort( + `${sourcePath}[${documentIndexInFile}]: no top-level \`metadata:\` mapping key to overlay onto`, + ); + } + + // The indent metadata's own children use, read from the document rather than assumed. + let childIndent = -1; + for (let index = metadataLine + 1; index < range.end; index += 1) { + const line = lines[index] as string; + if (line.trim() === '') continue; + const indent = leadingSpaces(line); + if (indent === 0) break; + childIndent = indent; + break; + } + if (childIndent <= 0) { + throw new OverlayAbort( + `${sourcePath}[${documentIndexInFile}]: \`metadata:\` has no indented children, so its child indent cannot be read`, + ); + } + + const quoted = `'${annotationValue.replaceAll("'", "''")}'`; + + // An existing `annotations:` mapping at metadata's child indent, if there is one. + // Commented-out lines (`# annotations:`) do not match, which is the point: two corpus + // descriptors carry exactly that as a documentation example. + const annotationsPattern = new RegExp(`^ {${childIndent}}annotations:\\s*$`); + let annotationsLine = -1; + for (let index = metadataLine + 1; index < range.end; index += 1) { + const line = lines[index] as string; + if (line.trim() === '') continue; + if (leadingSpaces(line) === 0) break; + if (leadingSpaces(line) === childIndent && annotationsPattern.test(line)) { + annotationsLine = index; + break; + } + } + + const inserted = + annotationsLine === -1 + ? { + at: metadataLine + 1, + lines: [ + `${' '.repeat(childIndent)}annotations:`, + `${' '.repeat(childIndent * 2)}${OWNED_PATHS_ANNOTATION}: ${quoted}`, + ], + } + : { + at: annotationsLine + 1, + lines: [`${' '.repeat(childIndent * 2)}${OWNED_PATHS_ANNOTATION}: ${quoted}`], + }; + + const overlaid = [ + ...lines.slice(0, inserted.at), + ...inserted.lines, + ...lines.slice(inserted.at), + ].join('\n'); + + assertOverlayIsAdditiveOnly(sourcePath, text, overlaid, documentIndexInFile, annotationValue); + return overlaid; +} + +/** + * Confirm the overlay added exactly one annotation and changed nothing else. + * + * Checked with the **generator's own** descriptor reader rather than a second parser, so + * the thing that validates the overlay is the thing that will consume it. A second + * notion of "same document" could disagree with the one that matters. + */ +function assertOverlayIsAdditiveOnly( + sourcePath: string, + pristine: string, + overlaid: string, + documentIndexInFile: number, + annotationValue: string, +): void { + const before = readDescriptorDocuments(sourcePath, pristine); + const after = readDescriptorDocuments(sourcePath, overlaid); + + if (after.length !== before.length) { + throw new OverlayAbort( + `${sourcePath}: overlay changed the document count from ${before.length} to ${after.length}`, + ); + } + for (const [index, document] of after.entries()) { + if (document.rejection !== undefined) { + throw new OverlayAbort( + `${sourcePath}[${index}]: the overlaid text no longer reads cleanly — ${document.rejection.detail}`, + ); + } + const pristineRaw = structuredClone((before[index]?.raw ?? null) as unknown) as Record< + string, + unknown + > | null; + const overlaidRaw = structuredClone(document.raw as unknown) as Record | null; + + if (index === documentIndexInFile) { + const metadata = (overlaidRaw?.['metadata'] ?? {}) as Record; + const annotations = (metadata['annotations'] ?? {}) as Record; + if (annotations[OWNED_PATHS_ANNOTATION] !== annotationValue) { + throw new OverlayAbort( + `${sourcePath}[${index}]: the injected annotation did not decode back to the overlay value`, + ); + } + // Remove exactly the injected key (and the annotations map, when the overlay + // created it) and require the remainder to be identical to the pristine document. + delete annotations[OWNED_PATHS_ANNOTATION]; + const pristineMetadata = (pristineRaw?.['metadata'] ?? {}) as Record; + if (!Object.hasOwn(pristineMetadata, 'annotations') && Object.keys(annotations).length === 0) { + delete metadata['annotations']; + } + } + + if (JSON.stringify(overlaidRaw) !== JSON.stringify(pristineRaw)) { + throw new OverlayAbort( + `${sourcePath}[${index}]: the overlay changed the document beyond adding one annotation`, + ); + } + } +} + +// ── The comparison kernel — pure, and the part that renders the verdict ─────────────── + +export interface ComparisonOutcome { + readonly pass: boolean; + readonly falsePositives: readonly Finding[]; + readonly falseNegatives: readonly Finding[]; + readonly otherMismatches: readonly Finding[]; + /** Entities the generator emitted that the expectation set does not cover. */ + readonly entitiesOutsideTheExpectationSet: readonly ObservedEntity[]; +} + +/** + * Diff derived ownership against the frozen expectations. + * + * Pure: it takes the two sides as data and returns the verdict. Nothing here reads a + * file, so the FAIL path is reachable in a test without mutating anything on disk — which + * is what makes ADR-0016's "observed failing" requirement satisfiable for this gate. + * + * **Entities outside the expectation set are not automatically findings.** The 24 selected + * documents do not exhaust their files: one selected file holds a second, unselected + * document, which the generator reads because a manifest names the *file*. Such an entity + * carries no overlay, so it must derive nothing. It is a false positive **only** if it + * derives a non-empty path set — an ownership claim no frozen expectation licenses. This + * distinction is the descriptor-file-count / entity-document-count trap, and collapsing it + * either way would be wrong: failing on the sibling's mere existence would report a defect + * that did not happen, and ignoring it entirely would let an unlicensed claim through. + */ +export function compare( + expected: readonly ExpectedEntity[], + observed: readonly ObservedEntity[], + derivedPathPatterns: readonly string[], +): ComparisonOutcome { + const falsePositives: Finding[] = []; + const falseNegatives: Finding[] = []; + const otherMismatches: Finding[] = []; + + const observedById = new Map(observed.map((entity) => [entity.canonicalId, entity] as const)); + const expectedIds = new Set(expected.map((entity) => entity.canonicalId)); + + for (const entity of expected) { + const actual = observedById.get(entity.canonicalId); + if (actual === undefined) { + falseNegatives.push({ + kind: 'false-negative', + reason: REASON_ENTITY_MISSING, + canonicalId: entity.canonicalId, + detail: `expected ${entity.expectedPaths.length} derived path(s) from ${entity.sourcePath}[${entity.documentIndexInFile}]; the generator emitted no entity with this canonical id`, + }); + continue; + } + + const derived = new Set(actual.derivedPaths); + for (const path of entity.expectedPaths) { + if (!derived.has(path)) { + falseNegatives.push({ + kind: 'false-negative', + reason: REASON_PATH_NOT_DERIVED, + canonicalId: entity.canonicalId, + detail: `expected ${JSON.stringify(path)}; derivedPaths = ${JSON.stringify(actual.derivedPaths)}`, + }); + } + } + + const expectedSet = new Set(entity.expectedPaths); + for (const path of actual.derivedPaths) { + if (!expectedSet.has(path)) { + falsePositives.push({ + kind: 'false-positive', + reason: REASON_PATH_NOT_EXPECTED, + canonicalId: entity.canonicalId, + detail: `derived ${JSON.stringify(path)}; expectedPaths = ${JSON.stringify(entity.expectedPaths)}`, + }); + } + } + + if (actual.ownershipState !== entity.ownershipState) { + otherMismatches.push({ + kind: 'other-mismatch', + reason: REASON_OWNERSHIP_STATE, + canonicalId: entity.canonicalId, + detail: `expected ${entity.ownershipState}, derived ${actual.ownershipState}`, + }); + } + + // Order is part of the expectation, not decoration: `owned-paths-annotation.md` §3 + // requires an `explicit-paths` derivation to be `compareCodeUnits`-sorted, and the + // whole ADR-0020 clause-6 re-freeze exists because the spike's oracle recorded input + // order. Same members in the wrong order is a mismatch and fails the gate. + const sameMembers = + actual.derivedPaths.length === entity.expectedPaths.length && + [...derived].every((path) => expectedSet.has(path)); + const sameOrder = + actual.derivedPaths.length === entity.expectedPaths.length && + entity.expectedPaths.every((path, index) => actual.derivedPaths[index] === path); + if (sameMembers && !sameOrder) { + otherMismatches.push({ + kind: 'other-mismatch', + reason: REASON_PATH_ORDER, + canonicalId: entity.canonicalId, + detail: `expected ${JSON.stringify(entity.expectedPaths)}, derived ${JSON.stringify(actual.derivedPaths)}`, + }); + } + + if ( + actual.sourcePath !== entity.sourcePath || + actual.documentIndexInFile !== entity.documentIndexInFile + ) { + otherMismatches.push({ + kind: 'other-mismatch', + reason: REASON_SOURCE_DOCUMENT, + canonicalId: entity.canonicalId, + detail: `freeze records ${entity.sourcePath}[${entity.documentIndexInFile}], envelope records ${actual.sourcePath}[${actual.documentIndexInFile}]`, + }); + } + } + + const outside = observed.filter((entity) => !expectedIds.has(entity.canonicalId)); + for (const entity of outside) { + if (entity.derivedPaths.length > 0) { + falsePositives.push({ + kind: 'false-positive', + reason: REASON_UNEXPECTED_OWNERSHIP, + canonicalId: entity.canonicalId, + detail: `${entity.sourcePath}[${entity.documentIndexInFile}] carries no frozen expectation yet derived ${JSON.stringify(entity.derivedPaths)}`, + }); + } + } + + // The oracle's 25-element pattern union, re-derived from what was actually produced. + const union = [ + ...new Set( + expected.flatMap((entity) => observedById.get(entity.canonicalId)?.derivedPaths ?? []), + ), + ].sort(compareCodeUnits); + if ( + union.length !== derivedPathPatterns.length || + union.some((pattern, index) => pattern !== derivedPathPatterns[index]) + ) { + otherMismatches.push({ + kind: 'other-mismatch', + reason: REASON_PATTERN_UNION, + canonicalId: '(corpus-wide)', + detail: `oracle records ${derivedPathPatterns.length} patterns, output yields ${union.length}: ${JSON.stringify(union)}`, + }); + } + + return { + pass: falsePositives.length === 0 && falseNegatives.length === 0 && otherMismatches.length === 0, + falsePositives, + falseNegatives, + otherMismatches, + entitiesOutsideTheExpectationSet: outside, + }; +} + +// ── Running the generator over the corpus ───────────────────────────────────────────── + +/** A deliberate corruption of the comparison input. Used only by T089's negative case. */ +export interface OverlayMutation { + readonly sourcePath: string; + readonly documentIndexInFile: number; + readonly annotationValue: string; +} + +/** + * T089's retained failing input, as a named constant rather than free-form CLI text. + * + * ADR-0016 clause 2 requires the failing input be kept "as a permanent negative case, not + * a throwaway. The artifact that proves the check works is the case that makes it fail." + * A `--mutate ` flag would make the retained artifact a shell invocation + * somebody has to reproduce exactly; a fixed constant makes it a line of code. + * + * **What it changes, and what it cannot.** It replaces one overlay annotation *value* — + * the corpus side of the comparison — with a plausible-looking near-miss: the two frozen + * patterns for `backstage-plugin-adr-backend` become one right and one wrong. It cannot + * touch the frozen expectations; nothing in this file writes to `evidence/`, and + * `scripts/check-freeze-hashes.ts` would fail the build if anything did. + * + * The near-miss is deliberate. A wholesale replacement would fail on everything at once + * and would not show that the comparison discriminates; this yields exactly one false + * negative (`packages/cli/**` expected, not derived) and exactly one false positive + * (`packages/clx/**` derived, not expected) from a single altered character. + */ +export const T089_MUTATION: OverlayMutation = { + sourcePath: 'workspaces/adr/plugins/adr-backend/catalog-info.yaml', + documentIndexInFile: 0, + annotationValue: '["packages/core/**","packages/clx/**"]', +}; + +export interface RunResult { + readonly envelope: SnapshotEnvelope; + readonly observed: readonly ObservedEntity[]; + /** sha256 of each overlaid descriptor, as fed to the generator. Step (b)'s own hashes. */ + readonly overlaidDigests: Readonly>; +} + +async function git(args: readonly string[], cwd: string): Promise { + const proc = Bun.spawn(['git', ...args], { + cwd, + stdout: 'pipe', + stderr: 'pipe', + env: { + ...Bun.env, + GIT_TERMINAL_PROMPT: '0', + GIT_CONFIG_GLOBAL: '/dev/null', + GIT_CONFIG_SYSTEM: '/dev/null', + }, + }); + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]); + if (exitCode !== 0) throw new Error(`git ${args.join(' ')} failed: ${stderr.trim()}`); + return stdout.trim(); +} + +/** + * Generate over the frozen accept corpus, with the overlay applied into a temporary run + * directory that is removed before this function returns. + * + * # Why the run directory declares the corpus's identity + * + * `repository/identity.ts` compares the manifest's declared `{id, revision}` against the + * checkout's observed identity. The descriptors here came from + * `github.com/backstage/community-plugins` at the pinned commit, and their bytes were + * verified against the content address that commit fixes (`scripts/vendor-accept-corpus.ts`), + * so that pair *is* their true identity and declaring anything else would be the false + * statement. + * + * A temporary git checkout cannot be made to report a foreign commit as its `HEAD`, so the + * identity is supplied through `GenerationRequest.observedRepositoryState` — which + * `pipeline.ts` documents as "the checkout's observed identity, when the caller has already + * read it… a **value**, never a function", precisely so a caller can supply data without + * supplying behaviour. Every other stage runs exactly as it does in production. + * + * # Nothing is written into the vendored corpus + * + * The pristine tree is read and never written. The overlaid bytes exist only inside the + * temporary directory, which is deleted in a `finally`. + */ +export async function generateOverCorpus( + repoRoot: string, + frozen: FrozenSide, + overlay: readonly OverlayEntry[], + mutation?: OverlayMutation, +): Promise { + const sourcePaths = [...new Set(frozen.entities.map((entity) => entity.sourcePath))].sort( + compareCodeUnits, + ); + + const effectiveOverlay = [...overlay]; + if (mutation !== undefined) { + const index = effectiveOverlay.findIndex( + (entry) => + entry.sourcePath === mutation.sourcePath && + entry.documentIndexInFile === mutation.documentIndexInFile, + ); + if (index === -1) { + throw new Error(`mutation targets ${mutation.sourcePath}, which the overlay does not cover`); + } + effectiveOverlay[index] = { ...mutation }; + } + + const run = await mkdtemp(join(tmpdir(), 'adrkit-clause5-step-b-')); + try { + await git(['init', '-q', '-b', 'main'], run); + await git(['remote', 'add', 'origin', `https://${frozen.repository}.git`], run); + + const overlaidDigests: Record = {}; + const sources: { path: string; digestAlgorithm: 'sha256'; digest: string }[] = []; + + for (const sourcePath of sourcePaths) { + const pristine = await readFile(join(repoRoot, CORPUS_DIR, sourcePath), 'utf8'); + let text = pristine; + for (const entry of effectiveOverlay.filter((one) => one.sourcePath === sourcePath)) { + text = applyOverlay(sourcePath, text, entry.documentIndexInFile, entry.annotationValue); + } + + const destination = join(run, sourcePath); + await mkdir(dirname(destination), { recursive: true }); + await writeFile(destination, text, 'utf8'); + + const digest = sha256Hex(new TextEncoder().encode(text)); + overlaidDigests[sourcePath] = digest; + sources.push({ path: sourcePath, digestAlgorithm: 'sha256', digest }); + } + + const manifestPath = join(run, 'input-manifest.json'); + await writeFile( + manifestPath, + `${JSON.stringify( + { + manifestSchemaVersion: '1', + requestedSnapshotSchemaVersion: '1', + requiredCapabilities: ['pathOwnership'], + repository: { id: frozen.repository, revision: frozen.commit }, + sources, + }, + null, + 2, + )}\n`, + 'utf8', + ); + + const outcome = await runGeneration({ + manifestPath, + checkoutRoot: run, + provenance: allMaintainerOverlay(sources.map((source) => source.path)), + observedRepositoryState: { + remoteRaw: `https://${frozen.repository}.git`, + head: frozen.commit, + }, + }); + + if (!outcome.ok) { + throw new Error( + `the generator rejected the frozen accept corpus: ${outcome.failure.triggerClass} — ${outcome.failure.detail}`, + ); + } + + return { + envelope: outcome.envelope, + observed: outcome.envelope.entities.map((entity) => ({ + canonicalId: entity.identity.canonicalId, + ownershipState: entity.ownershipState, + derivedPaths: entity.derivedPaths, + sourcePath: entity.sourceDocument.sourcePath, + documentIndexInFile: entity.sourceDocument.documentIndexInFile, + })), + overlaidDigests, + }; + } finally { + await rm(run, { recursive: true, force: true }); + } +} + +// ── The report ──────────────────────────────────────────────────────────────────────── + +export interface ComparisonReport { + readonly verdict: 'PASS' | 'FAIL'; + readonly outcome: ComparisonOutcome; + readonly run: RunResult; + readonly frozen: FrozenSide; +} + +export async function compareAcceptCorpus( + repoRoot: string, + mutation?: OverlayMutation, +): Promise { + const frozen = readFrozenSide( + JSON.parse(await readFile(join(repoRoot, FREEZE_PATH), 'utf8')) as unknown, + JSON.parse(await readFile(join(repoRoot, ORACLE_PATH), 'utf8')) as unknown, + ); + const overlay = readOverlay( + JSON.parse(await readFile(join(repoRoot, OVERLAY_PATH), 'utf8')) as unknown, + ); + + const run = await generateOverCorpus(repoRoot, frozen, overlay, mutation); + const outcome = compare(frozen.entities, run.observed, frozen.derivedPathPatterns); + + return { verdict: outcome.pass ? 'PASS' : 'FAIL', outcome, run, frozen }; +} + +/** The diff report, as written to `evidence/comparison/diff-report.json`. */ +export function renderDiffReport(report: ComparisonReport): Record { + const { frozen, outcome, run } = report; + return { + '//': + 'T088 — ADR-0020 clause 5 step (b): the post-output comparison. Derived ownership for every ' + + 'annotated entity in the frozen accept corpus, diffed against the frozen expectations at zero ' + + 'false positives and zero false negatives. Written by scripts/compare-accept-corpus.ts. The ' + + 'expectations were NOT amended to fit this output, and could not have been: ' + + 'scripts/check-freeze-hashes.ts fails the build on any drift in the frozen trees.', + task: 'T088', + barrierSide: 'BEHIND', + discharges: ['FR-056', 'SC-011'], + verdict: report.verdict, + corpusRef: { repository: frozen.repository, commit: frozen.commit }, + frozenInputs: { + 'accept-corpus-freeze/accept-corpus-freeze.json': frozen.freezeContentHash, + 'frozen-expectations/frozen-expectation-set.json': frozen.oracleContentHash, + note: + 'Read, cross-checked against each other, and never written. The freeze itself states that ' + + 'the two artifacts are two views of one freeze and that a divergence between them is a ' + + 'freeze failure; this harness refuses to run if they disagree.', + }, + counts: { + expectedEntities: frozen.entities.length, + envelopeEntities: run.observed.length, + entitiesOutsideTheExpectationSet: outcome.entitiesOutsideTheExpectationSet.length, + falsePositives: outcome.falsePositives.length, + falseNegatives: outcome.falseNegatives.length, + otherMismatches: outcome.otherMismatches.length, + countingNote: + 'Descriptor FILE count and entity DOCUMENT count are different numbers. 24 files are named ' + + 'by the manifest; the envelope carries one entity per admissible YAML document in them, ' + + 'which is more. An entity outside the expectation set is a finding only if it derived a ' + + 'non-empty path set.', + }, + findings: { + falsePositives: outcome.falsePositives, + falseNegatives: outcome.falseNegatives, + otherMismatches: outcome.otherMismatches, + }, + entitiesOutsideTheExpectationSet: outcome.entitiesOutsideTheExpectationSet.map((entity) => ({ + canonicalId: entity.canonicalId, + ownershipState: entity.ownershipState, + derivedPaths: entity.derivedPaths, + sourcePath: entity.sourcePath, + documentIndexInFile: entity.documentIndexInFile, + })), + derivedOwnership: [...run.observed] + .filter((entity) => + frozen.entities.some((expected) => expected.canonicalId === entity.canonicalId), + ) + .sort((a, b) => compareCodeUnits(a.canonicalId, b.canonicalId)) + .map((entity) => ({ + canonicalId: entity.canonicalId, + ownershipState: entity.ownershipState, + derivedPaths: entity.derivedPaths, + sourcePath: entity.sourcePath, + documentIndexInFile: entity.documentIndexInFile, + })), + whatThisDoesAndDoesNotEstablish: { + establishes: + 'That the generator\u2019s derived ownership over this frozen corpus agrees, exactly and in ' + + 'order, with a maintainer-authored expectation set frozen before any generator output existed.', + doesNotEstablish: [ + 'It does not establish correctness in any absolute sense. The expectations are the maintainer\u2019s own, hand-derived from frozen contracts; agreement between our implementation and our specification is not independent evidence that either is right.', + 'The envelope\u2019s self-digest establishes INTEGRITY, not correctness. A semantically wrong envelope can carry a perfectly valid self-digest (ADR-0020 clause 5, FR-058, SC-012).', + 'It says nothing about Backstage as a running system. The admissibility warrant is exactly what the four pinned validator predicates return at Backstage commit 1121a4facd9e321179d0402c3f355e4a649e84d9.', + 'It does not evidence that the mapping reflects anyone\u2019s actual ownership, that anyone else wants the annotation, or that adoption risk has fallen.', + 'ADR-0014 rung 1 only \u2014 not reference-verified, not externally validated, and no release is scheduled, prepared, or implied.', + ], + }, + }; +} + +/** + * The only CLI flag, and it exists solely so the FAIL path is observable from a shell. + * + * ADR-0016 will not accept a gate that has only ever been seen green. Passing this + * applies {@link T089_MUTATION} to the comparison **input** and writes no report — the + * report on disk always describes an unmutated run. + */ +export const OBSERVE_FAILING_FLAG = '--observe-failing'; + +/** + * T090 — step (b)'s **own** record: its own hashes, its own verdict. + * + * ADR-0020 clause 5 requires the pre-output freeze/audit and the post-output comparison to + * be "two distinct steps, each recording its own hashes and its own PASS/FAIL", and + * `spec.md` FR-057 adds that "neither may inherit the other's verdict". + * + * So this record does three things and inherits nothing: + * + * - It **recomputes** each frozen artifact's canonical content hash from the artifact and + * compares it to the recorded value. `data-model.md` §16: "an audit that transcribes the + * author's declared hash has verified nothing." The same applies here. + * - It records hashes that are step (b)'s **own** — of the vendored corpus bytes, of the + * overlaid bytes actually fed to the generator, and of the envelope that came out. None + * of these existed at step (a), which is what makes this a second step rather than a + * restatement of the first. + * - It renders its own verdict from its own comparison. + */ +export function renderStepBRecord( + report: ComparisonReport, + recomputedFrozenHashes: Readonly>, + vendoredCorpusSha256: Readonly>, +): Record { + return { + '//': + 'T090 — ADR-0020 clause 5 step (b)\u2019s own record. Step (b) inherits nothing from step (a): ' + + 'it recomputes the frozen hashes rather than trusting them, records hashes of its own that did ' + + 'not exist at step (a), and renders its own verdict. Written by scripts/compare-accept-corpus.ts.', + task: 'T090', + barrierSide: 'BEHIND', + discharges: ['FR-057 (step (b) half)'], + step: 'ADR-0020 clause 5 step (b) — post-output comparison', + verdict: report.verdict, + inheritsFromStepA: false, + whyItInheritsNothing: + 'FR-057: "The pre-output freeze/audit (FR-053, FR-054) and the post-output comparison (FR-056) ' + + 'MUST be recorded as two distinct steps, each recording its own hashes and its own PASS/FAIL. ' + + 'Neither may inherit the other\u2019s verdict." Step (a)\u2019s PASS is not evidence for this one, and ' + + 'this one\u2019s PASS is not evidence that step (a) was right \u2014 it is evidence that the generator ' + + 'agrees with what step (a) froze.', + recomputedFrozenHashes, + recomputationNote: + 'Recorded values are read from the artifacts; recomputed values are derived with the canonical ' + + 'form of evidence/README.md §3 (contentHash key removed, keys ascending by compareCodeUnits, ' + + 'array order preserved, no insignificant whitespace, UTF-8, no trailing newline). A `match: false` ' + + 'here is a freeze failure and fails this record\u2019s verdict.', + ownHashes: { + note: + 'Hashes that belong to step (b) and did not exist at step (a). The corpus was not vendored, no ' + + 'overlaid input existed, and no envelope existed when the freeze was written.', + vendoredCorpusSha256, + overlaidSourceDigests: report.run.overlaidDigests, + envelopeDigest: report.run.envelope.digest, + envelopeDigestMeans: + 'INTEGRITY, not correctness. A semantically wrong envelope can carry a perfectly valid ' + + 'self-digest (ADR-0020 clause 5, FR-058, SC-012). What speaks to the derivation is the ' + + 'comparison, and what the comparison speaks to is agreement with a maintainer-authored ' + + 'expectation set \u2014 not correctness in any absolute sense.', + }, + counts: { + expectedEntities: report.frozen.entities.length, + envelopeEntities: report.run.envelope.entities.length, + falsePositives: report.outcome.falsePositives.length, + falseNegatives: report.outcome.falseNegatives.length, + otherMismatches: report.outcome.otherMismatches.length, + }, + honesty: { + rung: 'ADR-0014 rung 1 only. Maintainer-owned verification, which is not external, third-party, or community adoption. Only the corpus DATA is third-party.', + release: 'No release is scheduled, prepared, or implied. ADR-0020 clause 9 defers both the vehicle and the decision to release at all to a later record.', + backstage: + 'No claim is made about Backstage as a running system. The admissibility warrant is exactly what the four pinned validator predicates return at Backstage commit 1121a4facd9e321179d0402c3f355e4a649e84d9.', + }, + }; +} + +/** + * T091 — the prohibition guard's record: the expectations were never amended. + * + * ADR-0020 clause 5: "the expectations are never amended to fit the output". A comparison + * that passes because the expectations moved is not a passing comparison, and prose cannot + * distinguish the two — so the values Phase B independently recorded are compared against + * the values recomputed now, across the whole of Phase E and Phase F. + * + * Phase B's values are read from `barrier-b-checkpoint.json`, which was written by the + * independent auditor session rather than by the freeze's author, so the baseline is not a + * value this phase could have chosen. + */ +export function renderExpectationsUnchanged( + phaseBHashes: Readonly>, + recomputed: Readonly>, +): Record { + const comparisons = FROZEN_ARTIFACTS.map((artifact) => { + const phaseB = phaseBHashes[artifact]; + const now = recomputed[artifact]; + return { + artifact, + phaseBRecordedHash: phaseB ?? null, + recomputedNow: now?.recomputed ?? null, + recordedInArtifactNow: now?.recorded ?? null, + unchangedSincePhaseB: phaseB !== undefined && now !== undefined && now.recomputed === phaseB, + selfConsistentNow: now?.match === true, + }; + }); + + return { + '//': + 'T091 — the clause-5 prohibition, enforced rather than asserted: the frozen expectations were ' + + 'never amended to fit the output. Phase B\u2019s hash values are read from barrier-b-checkpoint.json ' + + '(written by the independent auditor session, not by the freeze\u2019s author) and compared against ' + + 'values recomputed from the live artifacts now, after all of Phase E and Phase F.', + task: 'T091', + barrierSide: 'BEHIND', + discharges: [], + enforces: 'ADR-0020 clause 5 — "the expectations are never amended to fit the output"', + phaseBSource: `${CHECKPOINT_PATH} \u2192 confirmations.mechanism2_hashMatch.recordedHashes`, + alsoRecordedAt: + 'accept-corpus-freeze/selection-basis.md §8.4 carries the same two values in a table, independently written.', + comparisons, + allUnchanged: comparisons.every((entry) => entry.unchangedSincePhaseB && entry.selfConsistentNow), + whyThisIsNotCircular: + 'The recomputation derives each hash from the artifact\u2019s bytes; the baseline comes from a ' + + 'record written before any generator existed. An artifact edited to fit output would recompute ' + + 'to a different value than the one Phase B recorded, whatever its own contentHash field says. ' + + 'scripts/check-freeze-hashes.ts catches the narrower case where the bytes moved but the recorded ' + + 'hash did not, and has been observed genuinely failing on a one-byte mutation ' + + '(negative-cases/freeze-drift/).', + honesty: { + rung: 'ADR-0014 rung 1 only.', + scope: + 'This shows the expectations did not move. It does not show they were right \u2014 that is not a ' + + 'property any hash can carry.', + }, + }; +} + +/** Recompute every frozen artifact's canonical content hash, and compare to its recorded value. */ +async function recomputeFrozenHashes( + repoRoot: string, +): Promise> { + const out: Record = {}; + for (const artifact of FROZEN_ARTIFACTS) { + const parsed = JSON.parse( + await readFile(join(repoRoot, EVIDENCE_DIR, artifact), 'utf8'), + ) as Record; + const recorded = parsed['contentHash']; + if (typeof recorded !== 'string') throw new Error(`${artifact} carries no contentHash`); + const recomputed = canonicalHash(parsed); + out[artifact] = { recorded, recomputed, match: recorded === recomputed }; + } + return out; +} + +/** sha256 of every vendored descriptor, read from the pristine tree this run consumed. */ +async function vendoredCorpusHashes( + repoRoot: string, + sourcePaths: readonly string[], +): Promise> { + const out: Record = {}; + for (const sourcePath of sourcePaths) { + const bytes = new Uint8Array( + await Bun.file(join(repoRoot, CORPUS_DIR, sourcePath)).arrayBuffer(), + ); + out[sourcePath] = sha256Hex(bytes); + } + return out; +} + +/** Phase B's independently-recorded hash values, from the Barrier B checkpoint. */ +async function phaseBHashes(repoRoot: string): Promise> { + const checkpoint = JSON.parse(await readFile(join(repoRoot, CHECKPOINT_PATH), 'utf8')) as { + confirmations?: { mechanism2_hashMatch?: { recordedHashes?: Record } }; + }; + const recorded = checkpoint.confirmations?.mechanism2_hashMatch?.recordedHashes; + if (recorded === undefined) { + throw new Error( + `${CHECKPOINT_PATH} carries no confirmations.mechanism2_hashMatch.recordedHashes, so the ` + + 'Phase B baseline cannot be read. Refusing to substitute the current values for it.', + ); + } + return recorded; +} + +async function writeJson(path: string, value: unknown): Promise { + await mkdir(dirname(path), { recursive: true }); + await writeFile(path, `${JSON.stringify(value, null, 2)}\n`, 'utf8'); +} + +async function main(repoRoot: string, argv: readonly string[]): Promise { + const observeFailing = argv.includes(OBSERVE_FAILING_FLAG); + const report = await compareAcceptCorpus(repoRoot, observeFailing ? T089_MUTATION : undefined); + + if (observeFailing) { + console.log( + `compare-accept-corpus: ${OBSERVE_FAILING_FLAG} — the comparison INPUT was deliberately ` + + `mutated at ${T089_MUTATION.sourcePath}[${T089_MUTATION.documentIndexInFile}]. ` + + 'The frozen expectations were not touched, and nothing was written.', + ); + } else { + // The whole of step (b)'s evidence comes out of this one run. Hand-writing any of it + // would make the record a claim about a run rather than a product of one. + const recomputed = await recomputeFrozenHashes(repoRoot); + const corpusHashes = await vendoredCorpusHashes( + repoRoot, + [...new Set(report.frozen.entities.map((entity) => entity.sourcePath))].sort(compareCodeUnits), + ); + + await writeJson(join(repoRoot, DIFF_REPORT_PATH), renderDiffReport(report)); + await writeJson( + join(repoRoot, STEP_B_RECORD_PATH), + renderStepBRecord(report, recomputed, corpusHashes), + ); + await writeJson( + join(repoRoot, EXPECTATIONS_UNCHANGED_PATH), + renderExpectationsUnchanged(await phaseBHashes(repoRoot), recomputed), + ); + } + + const { falsePositives, falseNegatives, otherMismatches } = report.outcome; + console.log( + `compare-accept-corpus: ${report.verdict} — ${report.frozen.entities.length} expected entities, ` + + `${falsePositives.length} false positive(s), ${falseNegatives.length} false negative(s), ` + + `${otherMismatches.length} other mismatch(es)`, + ); + for (const finding of [...falseNegatives, ...falsePositives, ...otherMismatches]) { + console.error(` ${finding.kind} ${finding.canonicalId}: ${finding.reason} — ${finding.detail}`); + } + if (report.verdict === 'FAIL') process.exitCode = 1; +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { + await main(process.cwd(), process.argv.slice(2)); +} diff --git a/scripts/vendor-accept-corpus.test.ts b/scripts/vendor-accept-corpus.test.ts new file mode 100644 index 00000000..d7b49d86 --- /dev/null +++ b/scripts/vendor-accept-corpus.test.ts @@ -0,0 +1,347 @@ +/** + * T086a — checks on the vendored accept corpus and on the acquisition script. + * + * Two jobs, and they are different: + * + * 1. **The vendored tree is what it claims to be.** Every file's git blob id is + * recomputed from the bytes on disk and matched against the vendor manifest, and + * every file is confirmed to carry **no** `adrkit.io/owned-paths` annotation. The + * second check is the one that matters most: if the overlay ever got baked into the + * vendored files, ADR-0020 clause 5's "otherwise unmodified" would stop being provable + * and `data-model.md` §10's upstream/maintainer boundary would stop being legible. + * Nothing here needs a network. + * + * 2. **The acquisition step stays separated from generation.** FR-018 and FR-052 require + * the generator to need no network, no credential and no service, and to not degrade + * to a networked path when one is available. Fetching a corpus is a one-time + * acquisition, not part of a run — so this asserts, by scanning source, that nothing + * the adapter or the comparison harness reaches imports this script. + * + * Every rule here is driven against an input that must trip it, per ADR-0016. The + * verification helpers are pure, so a digest mismatch and a truncated tree are both + * reachable without a network and without touching anything on disk. + */ + +import { describe, expect, test } from 'bun:test'; +import { readFile, readdir } from 'node:fs/promises'; +import { join } from 'node:path'; +import { + importSpecifiers, + stripComments, +} from '../packages/adapters/catalog-backstage/test/source-scan.ts'; +import { + CORPUS_DIR, + DESCRIPTOR_BASENAME, + REASON_BLOB_DIGEST_MISMATCH, + REASON_DESCRIPTOR_COUNT, + REASON_PATH_ABSENT, + REASON_TREE_TRUNCATED, + VENDOR_MANIFEST_NAME, + VendorAbort, + gitBlobId, + isDescriptorPath, + planFromFreeze, + resolveSelected, + sha256Hex, + verifyBlob, +} from './vendor-accept-corpus.ts'; + +const REPO_ROOT = process.cwd(); +const CORPUS_ROOT = join(REPO_ROOT, CORPUS_DIR); +const FREEZE = join( + REPO_ROOT, + 'specs/010-catalog-backstage/evidence/accept-corpus-freeze/accept-corpus-freeze.json', +); + +interface VendorManifest { + readonly corpusRef: { readonly repository: string; readonly commit: string }; + readonly fileCount: number; + readonly files: readonly { + readonly path: string; + readonly gitBlobSha1: string; + readonly sha256: string; + readonly byteLength: number; + }[]; +} + +async function readManifest(): Promise { + return JSON.parse(await readFile(join(CORPUS_ROOT, VENDOR_MANIFEST_NAME), 'utf8')) as VendorManifest; +} + +async function readFreeze(): Promise> { + return JSON.parse(await readFile(FREEZE, 'utf8')) as Record; +} + +async function descriptorFiles(): Promise { + const entries = (await readdir(CORPUS_ROOT, { recursive: true })).map(String); + return entries + .filter((entry) => entry.endsWith(DESCRIPTOR_BASENAME)) + .map((entry) => entry.split('\\').join('/')) + .sort(); +} + +describe('T086a — the vendored corpus matches the pin the freeze records', () => { + test('the vendor manifest names the freeze\u2019s corpusRef, not some other pin', async () => { + const [manifest, freeze] = await Promise.all([readManifest(), readFreeze()]); + const corpusRef = freeze['corpusRef'] as { repository: string; commit: string }; + expect(manifest.corpusRef.repository).toBe(corpusRef.repository); + expect(manifest.corpusRef.commit).toBe(corpusRef.commit); + }); + + test('exactly the freeze\u2019s selected sourcePaths are vendored \u2014 no more, no fewer', async () => { + const freeze = await readFreeze(); + const selected = new Set( + (freeze['expectedPaths'] as { sourcePath: string }[]).map((entry) => entry.sourcePath), + ); + const vendored = new Set(await descriptorFiles()); + expect([...vendored].sort()).toEqual([...selected].sort()); + // `size` is the freeze's own recorded corpus size; the vendored FILE count matches it + // here only because each selected entity comes from a distinct file. Entity DOCUMENT + // count is a different number and is checked by the comparison harness, not here. + expect(vendored.size).toBe(freeze['size'] as number); + }); + + test('every vendored file hashes to the git blob id the manifest records', async () => { + const manifest = await readManifest(); + expect(manifest.files.length).toBe(manifest.fileCount); + expect(manifest.files.length).toBeGreaterThan(0); + + for (const file of manifest.files) { + const bytes = new Uint8Array(await Bun.file(join(CORPUS_ROOT, file.path)).arrayBuffer()); + expect(gitBlobId(bytes)).toBe(file.gitBlobSha1); + expect(sha256Hex(bytes)).toBe(file.sha256); + expect(bytes.byteLength).toBe(file.byteLength); + } + }); + + test('no vendored descriptor carries the overlay annotation \u2014 the boundary is on disk', async () => { + // The single most important property of the vendored tree. The maintainer-authored + // overlay lives in evidence/accept-corpus-freeze/overlay.json and is applied at + // generation time into an ephemeral run directory. A vendored file carrying it would + // make ADR-0020 clause 5's "otherwise unmodified" unprovable by digest and would + // erase the upstream/maintainer boundary data-model.md §10 exists to keep legible. + const files = await descriptorFiles(); + expect(files.length).toBeGreaterThan(0); + for (const file of files) { + const text = await readFile(join(CORPUS_ROOT, file), 'utf8'); + expect(text).not.toContain('adrkit.io/owned-paths'); + } + }); + + test('the freeze\u2019s overlay is still a separate artifact carrying every annotation', async () => { + const overlay = JSON.parse( + await readFile( + join(REPO_ROOT, 'specs/010-catalog-backstage/evidence/accept-corpus-freeze/overlay.json'), + 'utf8', + ), + ) as { overlay: readonly { annotationValue: string }[] }; + // 23, not 24: the freeze records one `annotation-absent` entity, which by definition + // has no overlay entry. Read from accept-corpus-freeze.json's + // overlayProvenance.entries and ownershipStatesExercised. + expect(overlay.overlay.length).toBe(23); + }); +}); + +describe('T086a — the plan is derived from the freeze, never hard-coded', () => { + test('planFromFreeze reads repository, commit, selected paths and the descriptor count', async () => { + const plan = planFromFreeze(await readFreeze()); + expect(plan.repository).toBe('github.com/backstage/community-plugins'); + expect(plan.commit).toBe('92e9e4e09c76cc57f3475029b73e5ec84498a459'); + expect(plan.selectedPaths.length).toBe(24); + // 156 exact-basename descriptor FILES at the pin, from the freeze's + // corpusFacts.descriptorFilesExactBasename. That is not the 167 entity DOCUMENTS, + // and not the 24 selected. + expect(plan.descriptorFilesExactBasename).toBe(156); + }); + + test('selectedPaths are in ascending code-unit order, so the manifest is content-ordered', async () => { + const plan = planFromFreeze(await readFreeze()); + const sorted = [...plan.selectedPaths].sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)); + expect(plan.selectedPaths).toEqual(sorted); + }); + + test('a freeze with no corpusRef is refused rather than defaulted', () => { + expect(() => planFromFreeze({ expectedPaths: [] })).toThrow(VendorAbort); + }); +}); + +describe('T086a — observed failing: each abort fires on an input built to trip it', () => { + const plan = { + repository: 'github.com/example/corpus', + commit: 'a'.repeat(40), + selectedPaths: ['one/catalog-info.yaml'], + descriptorFilesExactBasename: 2, + } as const; + + const tree = { + truncated: false, + tree: [ + { path: 'one/catalog-info.yaml', type: 'blob', sha: 'b'.repeat(40) }, + { path: 'two/catalog-info.yaml', type: 'blob', sha: 'c'.repeat(40) }, + { path: 'one', type: 'tree', sha: 'd'.repeat(40) }, + ], + } as const; + + test('the baseline resolves, so the failures below are not vacuous', () => { + const selected = resolveSelected(plan, tree); + expect(selected.map((entry) => entry.path)).toEqual(['one/catalog-info.yaml']); + }); + + test('a truncated tree listing aborts', () => { + expect(() => resolveSelected(plan, { ...tree, truncated: true })).toThrow(REASON_TREE_TRUNCATED); + }); + + test('a descriptor-file count that disagrees with the freeze aborts', () => { + expect(() => resolveSelected({ ...plan, descriptorFilesExactBasename: 3 }, tree)).toThrow( + REASON_DESCRIPTOR_COUNT, + ); + }); + + test('a selected path missing from the pinned tree aborts', () => { + expect(() => + resolveSelected({ ...plan, selectedPaths: ['three/catalog-info.yaml'] }, tree), + ).toThrow(REASON_PATH_ABSENT); + }); + + test('a blob whose recomputed id differs from the pinned id aborts, and the reason names both', () => { + const bytes = new TextEncoder().encode('kind: Component\n'); + const entry = { path: 'one/catalog-info.yaml', type: 'blob', sha: 'f'.repeat(40) }; + expect(() => verifyBlob(entry, bytes)).toThrow(REASON_BLOB_DIGEST_MISMATCH); + try { + verifyBlob(entry, bytes); + } catch (error) { + expect((error as Error).message).toContain('f'.repeat(40)); + expect((error as Error).message).toContain(gitBlobId(bytes)); + } + }); + + test('a blob whose recomputed id matches is accepted, so the check is not always-fail', () => { + const bytes = new TextEncoder().encode('kind: Component\n'); + const record = verifyBlob({ path: 'p', type: 'blob', sha: gitBlobId(bytes) }, bytes); + expect(record.gitBlobSha1).toBe(gitBlobId(bytes)); + expect(record.byteLength).toBe(bytes.byteLength); + }); + + test('gitBlobId agrees with git\u2019s own hash-object on a real vendored file', async () => { + // The recomputation is only worth anything if it is git's definition and not a + // plausible-looking variant. Checked against `git hash-object` itself. + const manifest = await readManifest(); + const file = manifest.files[0]; + expect(file).toBeDefined(); + const proc = Bun.spawn(['git', 'hash-object', file!.path], { + cwd: CORPUS_ROOT, + stdout: 'pipe', + stderr: 'pipe', + }); + const [stdout, exitCode] = await Promise.all([new Response(proc.stdout).text(), proc.exited]); + expect(exitCode).toBe(0); + expect(stdout.trim()).toBe(file!.gitBlobSha1); + }); + + test('isDescriptorPath matches the exact basename rule and nothing adjacent to it', () => { + expect(isDescriptorPath('catalog-info.yaml')).toBe(true); + expect(isDescriptorPath('a/b/catalog-info.yaml')).toBe(true); + expect(isDescriptorPath('a/catalog-info.yml')).toBe(false); + expect(isDescriptorPath('a/my-catalog-info.yaml')).toBe(false); + expect(isDescriptorPath('a/catalog-info.yaml.bak')).toBe(false); + }); +}); + +describe('T086a — acquisition is not reachable from generation (FR-018, FR-052)', () => { + /** Every `.ts` file under a directory, repository-relative and forward-slashed. */ + async function sources(directory: string): Promise { + const entries = (await readdir(join(REPO_ROOT, directory), { recursive: true })).map(String); + return entries + .filter((entry) => entry.endsWith('.ts')) + .map((entry) => `${directory}/${entry}`.split('\\').join('/')) + .sort(); + } + + /** + * Files this scan deliberately skips, because they contain the rule literals themselves + * and would match their own patterns. + * + * The same relief valve `test/source-scan.ts` documents and for the same reason: "a file + * that *states* a rule is indistinguishable to it from a file that *breaks* one. Adding + * an entry is the correct fix and is preferred over renaming around the scanner." The + * list is named and asserted rather than filtered inline, so it cannot grow quietly. + */ + const EXCLUDED_FROM_HOST_SCAN: readonly string[] = [ + // Must name the corpus host in order to assert that only one file names it. + 'scripts/vendor-accept-corpus.test.ts', + ]; + + test('the exclusion list is exactly one file, and that file exists', async () => { + expect(EXCLUDED_FROM_HOST_SCAN).toEqual(['scripts/vendor-accept-corpus.test.ts']); + expect(await Bun.file(join(REPO_ROOT, EXCLUDED_FROM_HOST_SCAN[0] as string)).exists()).toBe(true); + }); + + test('no adapter source or test IMPORTS the acquisition script', async () => { + // Import specifiers, not prose. Phase E's sc-009 test names this script in a doc + // comment explaining where the corpus came from, and a text scan cannot tell that + // apart from a real import. `stripComments` + `importSpecifiers` are the adapter's + // own scanning primitives — one authority for "what does this file import". + const files = await sources('packages/adapters/catalog-backstage'); + expect(files.length).toBeGreaterThan(50); + + const offenders: string[] = []; + for (const file of files) { + const code = stripComments(await readFile(join(REPO_ROOT, file), 'utf8')); + if (importSpecifiers(code).some((specifier) => specifier.includes('vendor-accept-corpus'))) { + offenders.push(file); + } + } + expect(offenders).toEqual([]); + }); + + test('the scan can see a violation, so an empty result means "looked and found nothing"', () => { + // ADR-0016 clause 3: a scan that silently examined nothing reports the same green as + // one that looked properly. Driven against a fixture that must trip it. + const fixture = "import { gitBlobId } from '../../../scripts/vendor-accept-corpus.ts';\n"; + expect( + importSpecifiers(stripComments(fixture)).some((one) => one.includes('vendor-accept-corpus')), + ).toBe(true); + // And a doc comment naming the script is NOT a violation, which is the distinction + // the prose-based version of this check got wrong. + const commentOnly = '/** see scripts/vendor-accept-corpus.ts */\nexport const x = 1;\n'; + expect( + importSpecifiers(stripComments(commentOnly)).some((one) => one.includes('vendor-accept-corpus')), + ).toBe(false); + }); + + test('the comparison harness does not import the acquisition script either', async () => { + const code = stripComments( + await readFile(join(REPO_ROOT, 'scripts/compare-accept-corpus.ts'), 'utf8'), + ); + const imports = importSpecifiers(code); + expect(imports.length).toBeGreaterThan(0); + for (const specifier of imports) { + expect(specifier).not.toContain('vendor-accept-corpus'); + } + }); + + test('the acquisition script imports nothing from the adapter, so the split is two-way', async () => { + const code = stripComments( + await readFile(join(REPO_ROOT, 'scripts/vendor-accept-corpus.ts'), 'utf8'), + ); + const imports = importSpecifiers(code); + expect(imports.length).toBeGreaterThan(0); + for (const specifier of imports) { + expect(specifier.startsWith('node:')).toBe(true); + } + }); + + test('exactly one script names the corpus host, and it is the acquisition script', async () => { + // A weaker guard than the import checks and stated as such: it catches a second + // fetcher being added beside the generator, not every conceivable route to a network. + const files = (await sources('scripts')).filter( + (file) => !EXCLUDED_FROM_HOST_SCAN.includes(file), + ); + const fetchers: string[] = []; + for (const file of files) { + const text = await readFile(join(REPO_ROOT, file), 'utf8'); + if (text.includes('https://api.github.com')) fetchers.push(file); + } + expect(fetchers).toEqual(['scripts/vendor-accept-corpus.ts']); + }); +}); diff --git a/scripts/vendor-accept-corpus.ts b/scripts/vendor-accept-corpus.ts new file mode 100644 index 00000000..7dc80d4c --- /dev/null +++ b/scripts/vendor-accept-corpus.ts @@ -0,0 +1,398 @@ +/** + * T086a — **one-time acquisition** of the frozen accept corpus's descriptor files. + * + * # Why this script exists + * + * `evidence/accept-corpus-freeze/accept-corpus-freeze.json` freezes the corpus's + * *metadata* — the repository, the commit, the 24 selected `sourcePath` values, the + * maintainer-authored overlay, and the expected path matches. It does **not** contain + * the descriptor files themselves, and `evidence/README.md` §4 is explicit that it must + * not: the freeze tree carries no `InputManifest` and no corpus the generator could + * read, because R5 mechanism 1 (input absence) depends on that. + * + * Phase E recorded the consequence rather than working around it + * (`packages/adapters/catalog-backstage/test/sc-009.test.ts`, SC-009 limb 2): the + * descriptors are not present in this repository, so no pass over the frozen accept + * corpus is possible. T088 diffs "every annotated entity in the frozen accept corpus" + * and cannot run without them either. + * + * This script closes that gap by vendoring the 24 selected descriptors verbatim. + * + * # What is vendored, and what deliberately is not + * + * **Pristine upstream bytes only.** ADR-0020 clause 5 requires the descriptors be + * "authored upstream and **otherwise unmodified**", and that is provable by digest only + * if what lands on disk is byte-identical to what upstream published. The + * maintainer-authored `adrkit.io/owned-paths` overlay is therefore **not** baked into + * these files. It stays where the freeze put it — `accept-corpus-freeze/overlay.json` — + * and is applied by the comparison harness at generation time, into an ephemeral run + * directory that is deleted afterwards. + * + * Pre-merging the two would destroy the boundary `data-model.md` §10's `provenance` + * field exists to make legible: no reader could tell by inspection which bytes are + * upstream and which are ours. The freeze already separates them; this preserves that. + * + * # What "verify against the digest" means here, stated precisely + * + * **The freeze records no per-descriptor digest.** Its only recorded hashes are the two + * artifact-level `contentHash` values (`evidence/README.md` §3), which cover the frozen + * metadata, not the corpus bytes. Asserting otherwise would be a claim about an artifact + * that does not carry it, so it is stated plainly instead. + * + * What the freeze *does* fix is a **commit pin**: `corpusRef.commit`. A git commit names + * exactly one tree, which names exactly one blob per path, and a git blob id is a + * content-addressed digest of that blob's bytes. Three checks follow from that, and all + * three abort rather than warn: + * + * 1. **Tree completeness.** The recursive tree listing must not be truncated. A + * truncated listing could silently omit a selected path, and "not found" would then be + * indistinguishable from "not fetched". + * 2. **Corpus-fact agreement.** The number of blobs whose basename is exactly + * `catalog-info.yaml` must equal `corpusFacts.descriptorFilesExactBasename` — a figure + * the freeze re-derived from this pin. A pin that moved, or a fetch that resolved + * somewhere else, changes this number. + * 3. **Per-blob content address.** For every selected path, the git blob id is + * **recomputed from the received bytes** — `sha1("blob " + byteLength + "\0" + bytes)` + * — and compared to the id the pinned tree records. The recomputation is over what + * arrived, never a transcription of what was claimed; `data-model.md` §16's rule ("an + * audit that transcribes the author's declared hash has verified nothing") applies to + * a fetch exactly as it applies to an audit. + * + * Any mismatch means the pin moved or the fetch is wrong. Neither is something to paper + * over, so this script exits non-zero and writes nothing. + * + * # The offline posture (FR-018, FR-052) is not weakened by this + * + * Fetching a corpus is an **acquisition step**, not part of a generation run. It happens + * once, by hand, and its output is committed. The generator remains offline, + * credential-free and network-free: it reads one local manifest and the local files that + * manifest names. + * + * The separation is kept structural rather than asserted. This module is not imported by + * the adapter, by the comparison harness, or by anything either of them reaches; + * `scripts/vendor-accept-corpus.test.ts` asserts exactly that, by scanning source rather + * than by trusting this paragraph. + * + * # Honesty + * + * Vendoring corpus **data** is not validation. Per ADR-0014's honesty rules and + * `evidence/README.md` §5, only the corpus data is third-party; the overlay, the expected + * paths, the audit and every check here are the maintainer's own. This work sits at + * ADR-0014 **rung 1**: not reference-verified (rung 2), not externally validated (rung 3). + * + * @see `specs/010-catalog-backstage/evidence/accept-corpus-freeze/accept-corpus-freeze.json` + * @see `docs/adr/0020-rescope-sc-010-and-authorize-work-toward-the-backstage-catalog-adapter.md` clause 5 + * @see `docs/adr/0014-stage-phase-landing-evidence-across-a-three-rung-validation-ladder.md` + */ + +import { mkdir, readFile, writeFile } from 'node:fs/promises'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +/** Repository-relative location of the vendored corpus. */ +export const CORPUS_DIR = 'specs/010-catalog-backstage/corpus'; + +/** The vendor manifest's file name, inside {@link CORPUS_DIR}. */ +export const VENDOR_MANIFEST_NAME = 'VENDOR-MANIFEST.json'; + +/** Repository-relative location of the freeze this script reads its inputs from. */ +export const FREEZE_PATH = + 'specs/010-catalog-backstage/evidence/accept-corpus-freeze/accept-corpus-freeze.json'; + +/** Exact basename a corpus descriptor file must have, per the freeze's selection rule. */ +export const DESCRIPTOR_BASENAME = 'catalog-info.yaml'; + +/** Reasons this script aborts. Each is asserted verbatim by the test. */ +export const REASON_TREE_TRUNCATED = + 'the pinned tree listing came back truncated, so a missing path cannot be distinguished from an unfetched one'; +export const REASON_DESCRIPTOR_COUNT = + 'the number of exact-basename descriptor files at the pin does not match the freeze-recorded corpusFacts.descriptorFilesExactBasename'; +export const REASON_PATH_ABSENT = 'a selected sourcePath is not a blob in the pinned tree'; +export const REASON_BLOB_DIGEST_MISMATCH = + 'the git blob id recomputed from the received bytes does not match the id the pinned tree records'; + +/** One vendored file, as recorded in the vendor manifest. */ +export interface VendoredFile { + /** Repository-relative path **inside the upstream corpus**, verbatim from the freeze. */ + readonly path: string; + /** The pinned tree's blob id, recomputed from the received bytes and matched against it. */ + readonly gitBlobSha1: string; + /** Lowercase-hex SHA-256 of the received bytes, for drift detection after vendoring. */ + readonly sha256: string; + readonly byteLength: number; +} + +export interface VendorPlan { + readonly repository: string; + readonly commit: string; + readonly selectedPaths: readonly string[]; + readonly descriptorFilesExactBasename: number; +} + +/** Anything this script refuses to proceed past. */ +export class VendorAbort extends Error { + constructor( + readonly reason: string, + detail: string, + ) { + super(`${reason}: ${detail}`); + this.name = 'VendorAbort'; + } +} + +interface TreeEntry { + readonly path: string; + readonly type: string; + readonly sha: string; +} + +interface TreeResponse { + readonly tree: readonly TreeEntry[]; + readonly truncated: boolean; +} + +/** + * Read the freeze and derive everything this script needs from it. + * + * Nothing about the corpus is hard-coded here: the repository, the commit, the selected + * paths and the descriptor-file count all come out of the frozen artifact, so a script + * run against a different freeze acquires that freeze's corpus and checks that freeze's + * facts. + */ +export function planFromFreeze(freeze: unknown): VendorPlan { + if (typeof freeze !== 'object' || freeze === null) { + throw new VendorAbort('freeze-unreadable', 'the freeze artifact is not a JSON object'); + } + const record = freeze as Record; + + const corpusRef = record['corpusRef'] as Record | undefined; + const repository = corpusRef?.['repository']; + const commit = corpusRef?.['commit']; + if (typeof repository !== 'string' || typeof commit !== 'string') { + throw new VendorAbort('freeze-unreadable', 'corpusRef.repository / corpusRef.commit missing'); + } + + const facts = record['corpusFacts'] as Record | undefined; + const descriptorFilesExactBasename = facts?.['descriptorFilesExactBasename']; + if (typeof descriptorFilesExactBasename !== 'number') { + throw new VendorAbort( + 'freeze-unreadable', + 'corpusFacts.descriptorFilesExactBasename missing, so the pin cannot be cross-checked', + ); + } + + const expected = record['expectedPaths']; + if (!Array.isArray(expected)) { + throw new VendorAbort('freeze-unreadable', 'expectedPaths is not an array'); + } + const paths = new Set(); + for (const entry of expected) { + const sourcePath = (entry as Record)['sourcePath']; + if (typeof sourcePath !== 'string') { + throw new VendorAbort('freeze-unreadable', 'an expectedPaths entry has no string sourcePath'); + } + paths.add(sourcePath); + } + + return { + repository, + commit, + // Ascending UTF-16 code-unit order, matching the repository's own comparator, so the + // vendor manifest's order is a function of content rather than of iteration. + selectedPaths: [...paths].sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)), + descriptorFilesExactBasename, + }; +} + +/** + * Git's own content address for a blob: `sha1("blob " + byteLength + "\0" + bytes)`. + * + * Recomputed from the bytes that arrived. This is the check that makes "otherwise + * unmodified" provable rather than asserted. + */ +export function gitBlobId(bytes: Uint8Array): string { + const header = new TextEncoder().encode(`blob ${bytes.byteLength}\0`); + const hasher = new Bun.CryptoHasher('sha1'); + hasher.update(header); + hasher.update(bytes); + return hasher.digest('hex'); +} + +export function sha256Hex(bytes: Uint8Array): string { + const hasher = new Bun.CryptoHasher('sha256'); + hasher.update(bytes); + return hasher.digest('hex'); +} + +/** Whether a tree path names a descriptor file under the freeze's exact-basename rule. */ +export function isDescriptorPath(path: string): boolean { + return path === DESCRIPTOR_BASENAME || path.endsWith(`/${DESCRIPTOR_BASENAME}`); +} + +/** + * Check the pinned tree against the freeze before a single blob is fetched. + * + * Ordered so the cheapest, most diagnostic failure comes first. Returns the tree entries + * for the selected paths, in the plan's order. + */ +export function resolveSelected(plan: VendorPlan, tree: TreeResponse): readonly TreeEntry[] { + if (tree.truncated) { + throw new VendorAbort(REASON_TREE_TRUNCATED, `${plan.repository}@${plan.commit}`); + } + + const blobs = tree.tree.filter((entry) => entry.type === 'blob'); + const descriptors = blobs.filter((entry) => isDescriptorPath(entry.path)); + if (descriptors.length !== plan.descriptorFilesExactBasename) { + throw new VendorAbort( + REASON_DESCRIPTOR_COUNT, + `freeze records ${plan.descriptorFilesExactBasename}, the pin yields ${descriptors.length}`, + ); + } + + const byPath = new Map(blobs.map((entry) => [entry.path, entry] as const)); + const selected: TreeEntry[] = []; + for (const path of plan.selectedPaths) { + const entry = byPath.get(path); + if (entry === undefined) throw new VendorAbort(REASON_PATH_ABSENT, path); + selected.push(entry); + } + return selected; +} + +/** + * Verify one fetched blob against the id the pinned tree records. + * + * Separated from the fetch so the check is testable without a network, which is the only + * way to observe it failing (ADR-0016). + */ +export function verifyBlob(entry: TreeEntry, bytes: Uint8Array): VendoredFile { + const recomputed = gitBlobId(bytes); + if (recomputed !== entry.sha) { + throw new VendorAbort( + REASON_BLOB_DIGEST_MISMATCH, + `${entry.path}: pinned tree records ${entry.sha}, received bytes hash to ${recomputed}`, + ); + } + return { + path: entry.path, + gitBlobSha1: recomputed, + sha256: sha256Hex(bytes), + byteLength: bytes.byteLength, + }; +} + +// ── Everything below this line touches the network, and nothing above it does. ──────── + +function apiHeaders(): Record { + const token = Bun.env['GH_TOKEN'] ?? Bun.env['GITHUB_TOKEN']; + return { + accept: 'application/vnd.github+json', + 'user-agent': 'adrkit-vendor-accept-corpus', + 'x-github-api-version': '2022-11-28', + ...(token === undefined || token === '' ? {} : { authorization: `Bearer ${token}` }), + }; +} + +async function api(url: string): Promise { + const response = await fetch(url, { headers: apiHeaders() }); + if (!response.ok) { + throw new VendorAbort('github-api-error', `${response.status} ${response.statusText} for ${url}`); + } + return (await response.json()) as T; +} + +async function fetchTree(plan: VendorPlan): Promise { + const [, owner, repo] = plan.repository.split('/'); + return await api( + `https://api.github.com/repos/${owner}/${repo}/git/trees/${plan.commit}?recursive=1`, + ); +} + +async function fetchBlob(plan: VendorPlan, sha: string): Promise { + const [, owner, repo] = plan.repository.split('/'); + const blob = await api<{ content?: string; encoding?: string }>( + `https://api.github.com/repos/${owner}/${repo}/git/blobs/${sha}`, + ); + if (blob.encoding !== 'base64' || typeof blob.content !== 'string') { + throw new VendorAbort('github-api-error', `blob ${sha} did not come back base64-encoded`); + } + return Uint8Array.from(Buffer.from(blob.content, 'base64')); +} + +async function main(repoRoot: string): Promise { + const freeze = JSON.parse(await readFile(join(repoRoot, FREEZE_PATH), 'utf8')) as unknown; + const plan = planFromFreeze(freeze); + + console.log(`vendor-accept-corpus: pin ${plan.repository}@${plan.commit}`); + const tree = await fetchTree(plan); + const selected = resolveSelected(plan, tree); + console.log( + `vendor-accept-corpus: pin verified — ${plan.descriptorFilesExactBasename} descriptor files, ` + + `${selected.length} selected`, + ); + + // Fetch and verify EVERY blob before writing ANY of them. A partial vendoring is a + // corpus that looks complete and is not, which is worse than none at all. + const fetched: { readonly entry: TreeEntry; readonly bytes: Uint8Array; readonly record: VendoredFile }[] = + []; + for (const entry of selected) { + const bytes = await fetchBlob(plan, entry.sha); + fetched.push({ entry, bytes, record: verifyBlob(entry, bytes) }); + } + + const corpusRoot = join(repoRoot, CORPUS_DIR); + for (const { entry, bytes } of fetched) { + const destination = join(corpusRoot, entry.path); + await mkdir(dirname(destination), { recursive: true }); + await writeFile(destination, bytes); + } + + const manifest = { + '//': `T086a — one-time acquisition record for the ADR-0020 clause-5 accept corpus. Written by scripts/vendor-accept-corpus.ts. The files below are PRISTINE upstream bytes: the maintainer-authored adrkit.io/owned-paths overlay is NOT applied here and lives in ${'evidence/accept-corpus-freeze/overlay.json'}, applied by the comparison harness into an ephemeral run directory.`, + task: 'T086a', + barrierSide: 'BEHIND', + corpusRef: { repository: plan.repository, commit: plan.commit }, + verification: { + freezeRecordsNoPerFileDigest: + 'The freeze records artifact-level contentHash values only (evidence/README.md §3); it records no per-descriptor digest. Stated rather than implied.', + whatWasVerified: [ + 'the recursive tree listing at the pin was not truncated', + `the pin yields exactly ${plan.descriptorFilesExactBasename} blobs whose basename is exactly ${DESCRIPTOR_BASENAME}, matching corpusFacts.descriptorFilesExactBasename`, + 'for every file below, the git blob id was RECOMPUTED from the received bytes and matched against the id the pinned tree records', + ], + gitBlobIdDefinition: 'sha1("blob " + byteLength + "\\0" + bytes), lowercase hex', + abortsOnMismatch: + 'Any mismatch exits non-zero and writes nothing. A mismatch means the pin moved or the fetch is wrong; neither is papered over.', + }, + honesty: { + rung: 'ADR-0014 rung 1 only. Vendoring corpus data is acquisition, not validation.', + thirdPartyBoundary: + 'Only the corpus DATA is third-party. The overlay, the expected paths, the audit and every check are the maintainer\u2019s own (evidence/README.md §5).', + offlinePosture: + 'This script is a one-time acquisition step and is not reachable from the generator or the comparison harness. Generation itself requires no network, no credential and no service (FR-018, FR-052).', + }, + fileCount: fetched.length, + files: fetched.map(({ record }) => record), + }; + + await writeFile( + join(corpusRoot, VENDOR_MANIFEST_NAME), + `${JSON.stringify(manifest, null, 2)}\n`, + 'utf8', + ); + + console.log(`vendor-accept-corpus: wrote ${fetched.length} descriptors to ${CORPUS_DIR}/`); +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { + try { + await main(process.cwd()); + } catch (error) { + if (error instanceof VendorAbort) { + console.error(`vendor-accept-corpus: ABORTED — ${error.message}`); + process.exitCode = 1; + } else { + throw error; + } + } +} diff --git a/specs/010-catalog-backstage/corpus/README.md b/specs/010-catalog-backstage/corpus/README.md new file mode 100644 index 00000000..4e4bcdd8 --- /dev/null +++ b/specs/010-catalog-backstage/corpus/README.md @@ -0,0 +1,81 @@ +# `corpus/` — the vendored ADR-0020 clause-5 accept corpus + +The 24 descriptor files the clause-5 accept-corpus freeze selects, vendored +**verbatim** from `github.com/backstage/community-plugins` at commit +`92e9e4e09c76cc57f3475029b73e5ec84498a459`. + +Acquired once, by `scripts/vendor-accept-corpus.ts`. `VENDOR-MANIFEST.json` +records what was fetched and how it was checked. + +## These files are pristine upstream bytes + +**There is no `adrkit.io/owned-paths` annotation anywhere in this directory, and +there must never be one.** + +ADR-0020 clause 5 requires the accept corpus's descriptors be "authored upstream +and **otherwise unmodified**". That is provable by digest only if what is on disk +is byte-identical to what upstream published. The maintainer-authored overlay +lives in `../evidence/accept-corpus-freeze/overlay.json` and is applied by +`scripts/compare-accept-corpus.ts` at generation time, into a temporary directory +that is deleted when the run ends. + +Keeping the two apart is not a style preference. `../data-model.md` §10's +`provenance` field exists precisely to make the upstream/maintainer boundary +legible; a pre-merged file would destroy it, because no reader could tell by +inspection which bytes are upstream and which are ours. The freeze already +separates them, and this preserves that separation on disk. + +`scripts/vendor-accept-corpus.test.ts` asserts it, file by file. + +## How the pin was checked + +The freeze records no per-descriptor digest — its only recorded hashes are the two +artifact-level `contentHash` values (`../evidence/README.md` §3). What it does fix +is a **commit pin**, and a git commit names exactly one tree, which names exactly +one blob per path, and a blob id is a content-addressed digest of that blob's +bytes. Three checks follow, and all three abort rather than warn: + +1. The recursive tree listing at the pin was **not truncated**, so "not found" + cannot be confused with "not fetched". +2. The pin yields exactly **156** blobs whose basename is exactly + `catalog-info.yaml`, matching the freeze's own + `corpusFacts.descriptorFilesExactBasename`. A pin that moved changes this + number. +3. For every vendored file, the git blob id was **recomputed from the bytes that + arrived** and matched against the id the pinned tree records. + +Three numbers that are easy to conflate, each read from +`../evidence/accept-corpus-freeze/accept-corpus-freeze.json` → `corpusFacts`: + +| Number | What it counts | +| --- | --- | +| **156** | descriptor **files** at the pin whose basename is exactly `catalog-info.yaml` | +| **167** | entity **documents** in those files — a file may hold several | +| **24** | documents selected into the frozen corpus, from 24 distinct files | + +A file count and a document count are different things. The 24 vendored files hold +more than 24 entity documents, and the comparison harness accounts for the +difference explicitly rather than assuming it away. + +## Re-acquiring + +```bash +bun run scripts/vendor-accept-corpus.ts +``` + +This is a **one-time acquisition step**, not part of a generation run. Generation +requires no network, no credential, and no service (FR-018, FR-052), and this +script is not reachable from the generator or from the comparison harness — +asserted by `scripts/vendor-accept-corpus.test.ts` rather than promised here. + +Re-running it over an unchanged pin rewrites the same bytes. If the pin has moved, +it aborts and writes nothing; that is a finding to investigate, not a state to +refresh past. + +## Standing constraints + +Only the corpus **data** here is third-party. The overlay, the expected paths, the +audit, and every check are the maintainer's own. Per ADR-0014's honesty rules none +of this may be described as external, third-party, or community validation, and +vendoring corpus data is acquisition rather than validation. ADR-0014 **rung 1** +only. diff --git a/specs/010-catalog-backstage/corpus/VENDOR-MANIFEST.json b/specs/010-catalog-backstage/corpus/VENDOR-MANIFEST.json new file mode 100644 index 00000000..98b0a9f3 --- /dev/null +++ b/specs/010-catalog-backstage/corpus/VENDOR-MANIFEST.json @@ -0,0 +1,171 @@ +{ + "//": "T086a — one-time acquisition record for the ADR-0020 clause-5 accept corpus. Written by scripts/vendor-accept-corpus.ts. The files below are PRISTINE upstream bytes: the maintainer-authored adrkit.io/owned-paths overlay is NOT applied here and lives in evidence/accept-corpus-freeze/overlay.json, applied by the comparison harness into an ephemeral run directory.", + "task": "T086a", + "barrierSide": "BEHIND", + "corpusRef": { + "repository": "github.com/backstage/community-plugins", + "commit": "92e9e4e09c76cc57f3475029b73e5ec84498a459" + }, + "verification": { + "freezeRecordsNoPerFileDigest": "The freeze records artifact-level contentHash values only (evidence/README.md §3); it records no per-descriptor digest. Stated rather than implied.", + "whatWasVerified": [ + "the recursive tree listing at the pin was not truncated", + "the pin yields exactly 156 blobs whose basename is exactly catalog-info.yaml, matching corpusFacts.descriptorFilesExactBasename", + "for every file below, the git blob id was RECOMPUTED from the received bytes and matched against the id the pinned tree records" + ], + "gitBlobIdDefinition": "sha1(\"blob \" + byteLength + \"\\0\" + bytes), lowercase hex", + "abortsOnMismatch": "Any mismatch exits non-zero and writes nothing. A mismatch means the pin moved or the fetch is wrong; neither is papered over." + }, + "honesty": { + "rung": "ADR-0014 rung 1 only. Vendoring corpus data is acquisition, not validation.", + "thirdPartyBoundary": "Only the corpus DATA is third-party. The overlay, the expected paths, the audit and every check are the maintainer’s own (evidence/README.md §5).", + "offlinePosture": "This script is a one-time acquisition step and is not reachable from the generator or the comparison harness. Generation itself requires no network, no credential and no service (FR-018, FR-052)." + }, + "fileCount": 24, + "files": [ + { + "path": "workspaces/acr/plugins/acr/catalog-info.yaml", + "gitBlobSha1": "a63f30f5ee916b87298037ee4b4108d9b60faad5", + "sha256": "220d3b8ea574bccce6d8fcd8297566d53b9f22a3f2e60586a5a60fb003544673", + "byteLength": 2496 + }, + { + "path": "workspaces/adr/plugins/adr-backend/catalog-info.yaml", + "gitBlobSha1": "8822cb5ee2170276bacbdfde91252807940f6cd3", + "sha256": "ef777c111a23d1ab714aeb2407e1403e9bfee84b61694dcc75c30534942cf527", + "byteLength": 229 + }, + { + "path": "workspaces/agent-forge/plugins/agent-forge/catalog-info.yaml", + "gitBlobSha1": "dfff155ae6badad8738574244e126b09ef0e4ddd", + "sha256": "7b7a57dd12c6e60dde41833037b3bd77f8fb3c8cb7b4a113f3d3c5aeb7540849", + "byteLength": 276 + }, + { + "path": "workspaces/airbrake/plugins/airbrake-backend/catalog-info.yaml", + "gitBlobSha1": "f4790ce4725bc28c8224e9d69295ad1d68da8b92", + "sha256": "f12c328e2c2a9e219f631b5a6a2bb179c4883dd3f1f01e2d12d5363afcd2af7f", + "byteLength": 244 + }, + { + "path": "workspaces/allure/plugins/allure/catalog-info.yaml", + "gitBlobSha1": "b50e306bdf5d0996c8c6d5297656aa8ed158c088", + "sha256": "70d8db4a983b8afac837e5179c25b032009c3ea08ff9607bf7d1de6d3f432afe", + "byteLength": 287 + }, + { + "path": "workspaces/analytics/plugins/analytics-module-ga4/catalog-info.yaml", + "gitBlobSha1": "88c8ecc13a00c465b361843eddbecf84e908b736", + "sha256": "117b126e92e30e49dd6444cb441519d1db898021b113755140b3da1d3987a58e", + "byteLength": 260 + }, + { + "path": "workspaces/apache-airflow/plugins/apache-airflow/catalog-info.yaml", + "gitBlobSha1": "ad6439f86e14daeacfb6cd8fc7db70f317f3cb5e", + "sha256": "f1bafbb958155dd813c1e10d3a7f8d3ab632c31adf0e535475204ee14a7d7fe6", + "byteLength": 241 + }, + { + "path": "workspaces/apollo-explorer/plugins/apollo-explorer/catalog-info.yaml", + "gitBlobSha1": "f906e548b468a5561695c952be8d0552e561d21f", + "sha256": "fa5d34aebc5214880e9c89490d5937cfcd48fdc6dae290ce67ade97755576275", + "byteLength": 243 + }, + { + "path": "workspaces/azure-devops/plugins/azure-devops-common/catalog-info.yaml", + "gitBlobSha1": "5c36592436e49f754828bc01f1effc50a4213a26", + "sha256": "ba961f94c5718d2bb9e61c4fa1ad4f22dbe384db99572e2c57d144f7bf3280d7", + "byteLength": 246 + }, + { + "path": "workspaces/azure-sites/plugins/azure-sites-backend/catalog-info.yaml", + "gitBlobSha1": "faa3c48563699ba16f3138b111cd4782418b14d5", + "sha256": "847a38e3525f0ed2e6bd291c1b05295043407704f6005414ce9e02a41d333e69", + "byteLength": 250 + }, + { + "path": "workspaces/badges/plugins/badges-backend/catalog-info.yaml", + "gitBlobSha1": "3f577c8643237f9193900339a5d77e866efa1ad0", + "sha256": "802effad65014eae49868aa6c58f33a28fd936ff3ee27cce1927a17f814855d0", + "byteLength": 329 + }, + { + "path": "workspaces/bazaar/plugins/bazaar-backend/catalog-info.yaml", + "gitBlobSha1": "b5b9c4f2ac5b7c835869c46c7bc3988ec84c40d9", + "sha256": "f29955f93d8d86ac590447eb5739d94a46dfd7ffa67c5d7778c55b3f30285871", + "byteLength": 240 + }, + { + "path": "workspaces/bitbucket-pull-requests/catalog-info.yaml", + "gitBlobSha1": "0f7c702e757cc35bdaf2df7c7d9cfa6b3ea49276", + "sha256": "caf63c13bc985053afd9177c6746e9a771ce85a1411991964af7b35efa9fbb0b", + "byteLength": 364 + }, + { + "path": "workspaces/bitrise/plugins/bitrise/catalog-info.yaml", + "gitBlobSha1": "35fb20ab7ad23067bfa2eeeedbf4e39c605f1e04", + "sha256": "3ac10cd165437e2635b6be691385350d592134d4446847b3f177aaaa1cb74a20", + "byteLength": 298 + }, + { + "path": "workspaces/bookmarks/plugins/bookmarks/examples/component/catalog-info.yaml", + "gitBlobSha1": "a39b943052a624fca23a455e43632299f9b3e0e0", + "sha256": "d2e57cf42130dd65243d41613dedcaafbd9dfa1dd54f3b31f37046dec426d68e", + "byteLength": 1069 + }, + { + "path": "workspaces/catalog/plugins/catalog-backend-module-codeowners/catalog-info.yaml", + "gitBlobSha1": "c824dedbe6ac9e5c5ce7707642b34f71032e4860", + "sha256": "1ac0c4ef588570bef861a2da35ddce18b7f66d8d555d4ed6549759ecb2537b01", + "byteLength": 344 + }, + { + "path": "workspaces/checkmarx/plugins/checkmarx-backend/catalog-info.yaml", + "gitBlobSha1": "19237900be1bee0b9fbe7e8bb140e5d53146e2a4", + "sha256": "9a746e20e8b82b8d5af9bae1c94bb87fe5dc6a32b6867eb5a59863077577b48e", + "byteLength": 246 + }, + { + "path": "workspaces/cicd-statistics/plugins/cicd-statistics-module-buildkite/catalog-info.yaml", + "gitBlobSha1": "7fa8569d39b47543db1916875eb573f0942beb4f", + "sha256": "6c6711934fed735785325e1e52838003c76cc6b3268d9f196aa96e9be77b82ca", + "byteLength": 347 + }, + { + "path": "workspaces/cloudbuild/plugins/cloudbuild/catalog-info.yaml", + "gitBlobSha1": "2b7abead617348054a0db63b61bf89d1b15cb9cf", + "sha256": "59049aa4342bdc43e284cd9e67e654ae9221a1223b938c3318d3585a401560a1", + "byteLength": 310 + }, + { + "path": "workspaces/code-climate/plugins/code-climate/catalog-info.yaml", + "gitBlobSha1": "74aaf8ce0a43f05ed9f202e087e3f4b5bd6073dc", + "sha256": "f20b9ba12a185e6abe23dda6b311c46f1a9378cce2daa1a80a039d51425c7ce4", + "byteLength": 237 + }, + { + "path": "workspaces/code-coverage/plugins/code-coverage-backend/catalog-info.yaml", + "gitBlobSha1": "81daf642710dc0a108cfff44bdb073f429eab968", + "sha256": "eca711137951f49b96a05840c1fceee73128657b13eb3d7536b3734175cc9471", + "byteLength": 337 + }, + { + "path": "workspaces/codescene/plugins/codescene/catalog-info.yaml", + "gitBlobSha1": "3003bb0628d8765689d61ade6889fe034986416d", + "sha256": "71ccfddf6aa1157746870fbb84ff7dcaff49ebd54e1eda8489bf0c4451312a35", + "byteLength": 231 + }, + { + "path": "workspaces/copilot/plugins/copilot-backend/catalog-info.yaml", + "gitBlobSha1": "d77ecc34a9af96a2cd2217e852886dceadf884e5", + "sha256": "642ed324ea086ac9c56ae952865f9ee65f576b36418b9d3f9bba6eddb2947177", + "byteLength": 242 + }, + { + "path": "workspaces/cost-insights/plugins/cost-insights-common/catalog-info.yaml", + "gitBlobSha1": "ab72402ab69a1d51a115a3b77e6c91fe73fd6c44", + "sha256": "9c98f90b3e03427f6c6dabd4b81125f6e4034c68fea6225e7110c99807a38c5a", + "byteLength": 321 + } + ] +} diff --git a/specs/010-catalog-backstage/corpus/workspaces/acr/plugins/acr/catalog-info.yaml b/specs/010-catalog-backstage/corpus/workspaces/acr/plugins/acr/catalog-info.yaml new file mode 100644 index 00000000..a63f30f5 --- /dev/null +++ b/specs/010-catalog-backstage/corpus/workspaces/acr/plugins/acr/catalog-info.yaml @@ -0,0 +1,51 @@ +# https://backstage.io/docs/features/software-catalog/descriptor-format#kind-component +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: backstage-community-acr + title: Azure Container Registry plugin + description: A Backstage plugin that displays information about your container images available in the Azure Container Registry + annotations: + backstage.io/source-location: url:https://github.com/backstage/community-plugins/tree/main/workspaces/acr/plugins/acr + backstage.io/view-url: https://github.com/backstage/community-plugins/tree/main/workspaces/acr/plugins/acr/catalog-info.yaml + backstage.io/edit-url: https://github.com/backstage/community-plugins/tree/main/workspaces/acr/plugins/acr/catalog-info.yaml + github.com/project-slug: backstage/community-plugins + github.com/team-slug: backstage-community/acr-plugin-maintainers + sonarqube.org/project-key: backstage-community_plugins + links: + - url: https://github.com/backstage/community-plugins/tree/main/workspaces/acr/plugins/acr + title: GitHub Source + icon: source + type: source +spec: + type: backstage-plugin + lifecycle: production + owner: owner-unknown + system: rhdh + subcomponentOf: backstage-community-plugins +--- +# https://backstage.io/docs/features/software-catalog/descriptor-format#kind-component +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: backstage-community-acr-frontend + title: '@backstage-community/plugin-acr' + description: A Backstage plugin that displays information about your container images available in the Azure Container Registry + annotations: + backstage.io/source-location: url:https://github.com/backstage/community-plugins/tree/main/workspaces/acr/plugins/acr + backstage.io/view-url: https://github.com/backstage/community-plugins/tree/main/workspaces/acr/plugins/acr/catalog-info.yaml + backstage.io/edit-url: https://github.com/backstage/community-plugins/tree/main/workspaces/acr/plugins/acr/catalog-info.yaml + github.com/project-slug: backstage/community-plugins + github.com/team-slug: backstage-community/acr-plugin-maintainers + sonarqube.org/project-key: backstage-community_plugins + links: + - url: https://github.com/backstage/community-plugins/tree/main/workspaces/acr/plugins/acr + title: GitHub Source + icon: source + type: source +spec: + type: backstage-frontend-plugin + lifecycle: production + owner: owner-unknown + system: rhdh + subcomponentOf: backstage-community-acr diff --git a/specs/010-catalog-backstage/corpus/workspaces/adr/plugins/adr-backend/catalog-info.yaml b/specs/010-catalog-backstage/corpus/workspaces/adr/plugins/adr-backend/catalog-info.yaml new file mode 100644 index 00000000..8822cb5e --- /dev/null +++ b/specs/010-catalog-backstage/corpus/workspaces/adr/plugins/adr-backend/catalog-info.yaml @@ -0,0 +1,9 @@ +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: backstage-plugin-adr-backend + title: '@backstage-community/plugin-adr-backend' +spec: + lifecycle: experimental + type: backstage-backend-plugin + owner: kuangp diff --git a/specs/010-catalog-backstage/corpus/workspaces/agent-forge/plugins/agent-forge/catalog-info.yaml b/specs/010-catalog-backstage/corpus/workspaces/agent-forge/plugins/agent-forge/catalog-info.yaml new file mode 100644 index 00000000..dfff155a --- /dev/null +++ b/specs/010-catalog-backstage/corpus/workspaces/agent-forge/plugins/agent-forge/catalog-info.yaml @@ -0,0 +1,10 @@ +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: backstage-plugin-agent-forge + title: '@backstage-community/plugin-agent-forge' + description: Chat plugin for AI agents +spec: + lifecycle: experimental + type: backstage-frontend-plugin + owner: maintainers diff --git a/specs/010-catalog-backstage/corpus/workspaces/airbrake/plugins/airbrake-backend/catalog-info.yaml b/specs/010-catalog-backstage/corpus/workspaces/airbrake/plugins/airbrake-backend/catalog-info.yaml new file mode 100644 index 00000000..f4790ce4 --- /dev/null +++ b/specs/010-catalog-backstage/corpus/workspaces/airbrake/plugins/airbrake-backend/catalog-info.yaml @@ -0,0 +1,9 @@ +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: backstage-plugin-airbrake-backend + title: '@backstage-community/plugin-airbrake-backend' +spec: + lifecycle: experimental + type: backstage-backend-plugin + owner: maintainers diff --git a/specs/010-catalog-backstage/corpus/workspaces/allure/plugins/allure/catalog-info.yaml b/specs/010-catalog-backstage/corpus/workspaces/allure/plugins/allure/catalog-info.yaml new file mode 100644 index 00000000..b50e306b --- /dev/null +++ b/specs/010-catalog-backstage/corpus/workspaces/allure/plugins/allure/catalog-info.yaml @@ -0,0 +1,10 @@ +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: backstage-plugin-allure + title: '@backstage-community/plugin-allure' + description: A Backstage plugin that integrates with Allure +spec: + lifecycle: experimental + type: backstage-frontend-plugin + owner: maintainers diff --git a/specs/010-catalog-backstage/corpus/workspaces/analytics/plugins/analytics-module-ga4/catalog-info.yaml b/specs/010-catalog-backstage/corpus/workspaces/analytics/plugins/analytics-module-ga4/catalog-info.yaml new file mode 100644 index 00000000..88c8ecc1 --- /dev/null +++ b/specs/010-catalog-backstage/corpus/workspaces/analytics/plugins/analytics-module-ga4/catalog-info.yaml @@ -0,0 +1,9 @@ +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: backstage-plugin-analytics-module-ga4 + title: '@backstage-community/plugin-analytics-module-ga4' +spec: + lifecycle: experimental + type: backstage-frontend-plugin-module + owner: maintainers diff --git a/specs/010-catalog-backstage/corpus/workspaces/apache-airflow/plugins/apache-airflow/catalog-info.yaml b/specs/010-catalog-backstage/corpus/workspaces/apache-airflow/plugins/apache-airflow/catalog-info.yaml new file mode 100644 index 00000000..ad6439f8 --- /dev/null +++ b/specs/010-catalog-backstage/corpus/workspaces/apache-airflow/plugins/apache-airflow/catalog-info.yaml @@ -0,0 +1,9 @@ +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: backstage-plugin-apache-airflow + title: '@backstage-community/plugin-apache-airflow' +spec: + lifecycle: experimental + type: backstage-frontend-plugin + owner: maintainers diff --git a/specs/010-catalog-backstage/corpus/workspaces/apollo-explorer/plugins/apollo-explorer/catalog-info.yaml b/specs/010-catalog-backstage/corpus/workspaces/apollo-explorer/plugins/apollo-explorer/catalog-info.yaml new file mode 100644 index 00000000..f906e548 --- /dev/null +++ b/specs/010-catalog-backstage/corpus/workspaces/apollo-explorer/plugins/apollo-explorer/catalog-info.yaml @@ -0,0 +1,9 @@ +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: backstage-plugin-apollo-explorer + title: '@backstage-community/plugin-apollo-explorer' +spec: + lifecycle: experimental + type: backstage-frontend-plugin + owner: maintainers diff --git a/specs/010-catalog-backstage/corpus/workspaces/azure-devops/plugins/azure-devops-common/catalog-info.yaml b/specs/010-catalog-backstage/corpus/workspaces/azure-devops/plugins/azure-devops-common/catalog-info.yaml new file mode 100644 index 00000000..5c365924 --- /dev/null +++ b/specs/010-catalog-backstage/corpus/workspaces/azure-devops/plugins/azure-devops-common/catalog-info.yaml @@ -0,0 +1,9 @@ +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: backstage-plugin-azure-devops-common + title: '@backstage-community/plugin-azure-devops-common' +spec: + lifecycle: experimental + type: backstage-common-library + owner: awanlin diff --git a/specs/010-catalog-backstage/corpus/workspaces/azure-sites/plugins/azure-sites-backend/catalog-info.yaml b/specs/010-catalog-backstage/corpus/workspaces/azure-sites/plugins/azure-sites-backend/catalog-info.yaml new file mode 100644 index 00000000..faa3c485 --- /dev/null +++ b/specs/010-catalog-backstage/corpus/workspaces/azure-sites/plugins/azure-sites-backend/catalog-info.yaml @@ -0,0 +1,9 @@ +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: backstage-plugin-azure-sites-backend + title: '@backstage-community/plugin-azure-sites-backend' +spec: + lifecycle: experimental + type: backstage-backend-plugin + owner: maintainers diff --git a/specs/010-catalog-backstage/corpus/workspaces/badges/plugins/badges-backend/catalog-info.yaml b/specs/010-catalog-backstage/corpus/workspaces/badges/plugins/badges-backend/catalog-info.yaml new file mode 100644 index 00000000..3f577c86 --- /dev/null +++ b/specs/010-catalog-backstage/corpus/workspaces/badges/plugins/badges-backend/catalog-info.yaml @@ -0,0 +1,10 @@ +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: backstage-plugin-badges-backend + title: '@backstage-community/plugin-badges-backend' + description: A Backstage backend plugin that generates README badges for your entities +spec: + lifecycle: experimental + type: backstage-backend-plugin + owner: maintainers diff --git a/specs/010-catalog-backstage/corpus/workspaces/bazaar/plugins/bazaar-backend/catalog-info.yaml b/specs/010-catalog-backstage/corpus/workspaces/bazaar/plugins/bazaar-backend/catalog-info.yaml new file mode 100644 index 00000000..b5b9c4f2 --- /dev/null +++ b/specs/010-catalog-backstage/corpus/workspaces/bazaar/plugins/bazaar-backend/catalog-info.yaml @@ -0,0 +1,9 @@ +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: backstage-plugin-bazaar-backend + title: '@backstage-community/plugin-bazaar-backend' +spec: + lifecycle: experimental + type: backstage-backend-plugin + owner: maintainers diff --git a/specs/010-catalog-backstage/corpus/workspaces/bitbucket-pull-requests/catalog-info.yaml b/specs/010-catalog-backstage/corpus/workspaces/bitbucket-pull-requests/catalog-info.yaml new file mode 100644 index 00000000..0f7c702e --- /dev/null +++ b/specs/010-catalog-backstage/corpus/workspaces/bitbucket-pull-requests/catalog-info.yaml @@ -0,0 +1,13 @@ +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: bitbucket-pull-requests + description: An example of a Backstage application. + # Example for optional annotations + # annotations: + # github.com/project-slug: backstage/backstage + # backstage.io/techdocs-ref: dir:. +spec: + type: website + owner: john@example.com + lifecycle: experimental diff --git a/specs/010-catalog-backstage/corpus/workspaces/bitrise/plugins/bitrise/catalog-info.yaml b/specs/010-catalog-backstage/corpus/workspaces/bitrise/plugins/bitrise/catalog-info.yaml new file mode 100644 index 00000000..35fb20ab --- /dev/null +++ b/specs/010-catalog-backstage/corpus/workspaces/bitrise/plugins/bitrise/catalog-info.yaml @@ -0,0 +1,10 @@ +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: backstage-plugin-bitrise + title: '@backstage-community/plugin-bitrise' + description: A Backstage plugin that integrates towards Bitrise +spec: + lifecycle: experimental + type: backstage-frontend-plugin + owner: sda-se-reviewers diff --git a/specs/010-catalog-backstage/corpus/workspaces/bookmarks/plugins/bookmarks/examples/component/catalog-info.yaml b/specs/010-catalog-backstage/corpus/workspaces/bookmarks/plugins/bookmarks/examples/component/catalog-info.yaml new file mode 100644 index 00000000..a39b9430 --- /dev/null +++ b/specs/010-catalog-backstage/corpus/workspaces/bookmarks/plugins/bookmarks/examples/component/catalog-info.yaml @@ -0,0 +1,23 @@ +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: example-website + + bookmarks: + 'Life story': https://docs.google.com/document/d/1qaLicIa3FZKyup4JXo9ivNgWDmkbX6-XBaQNfKeKjpw/mobilebasic + 'My cool gadgets and gizmos': + 'fortune cowsay lolcat': https://logonoff.co/projects/fcl/index.html + 'XP tour': https://logonoff.co/projects/windowsxptour/index.html + 'notepad': https://notepad.logonoff.co + 'Important documents': + 'Team sync notes': https://docs.google.com/document/d/1qaLicIa3FZKyup4JXo9ivNgWDmkbX6-XBaQNfKeKjpw/mobilebasic + 'Manifesto': + 'Agile manifesto': https://docs.google.com/document/d/1qaLicIa3FZKyup4JXo9ivNgWDmkbX6-XBaQNfKeKjpw/mobilebasic + 'Scrum manifesto': https://docs.google.com/document/d/1qaLicIa3FZKyup4JXo9ivNgWDmkbX6-XBaQNfKeKjpw/mobilebasic + 'Sprint planning': https://docs.google.com/document/d/1qaLicIa3FZKyup4JXo9ivNgWDmkbX6-XBaQNfKeKjpw/mobilebasic +spec: + type: website + lifecycle: experimental + owner: guests + system: examples + providesApis: [example-grpc-api] diff --git a/specs/010-catalog-backstage/corpus/workspaces/catalog/plugins/catalog-backend-module-codeowners/catalog-info.yaml b/specs/010-catalog-backstage/corpus/workspaces/catalog/plugins/catalog-backend-module-codeowners/catalog-info.yaml new file mode 100644 index 00000000..c824dedb --- /dev/null +++ b/specs/010-catalog-backstage/corpus/workspaces/catalog/plugins/catalog-backend-module-codeowners/catalog-info.yaml @@ -0,0 +1,10 @@ +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: backstage-plugin-catalog-backend-module-codeowners + title: '@backstage/plugin-catalog-backend-module-codeowners' + description: The codeowners backend module for the catalog plugin. +spec: + lifecycle: experimental + type: backstage-backend-plugin-module + owner: maintainers diff --git a/specs/010-catalog-backstage/corpus/workspaces/checkmarx/plugins/checkmarx-backend/catalog-info.yaml b/specs/010-catalog-backstage/corpus/workspaces/checkmarx/plugins/checkmarx-backend/catalog-info.yaml new file mode 100644 index 00000000..19237900 --- /dev/null +++ b/specs/010-catalog-backstage/corpus/workspaces/checkmarx/plugins/checkmarx-backend/catalog-info.yaml @@ -0,0 +1,9 @@ +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: backstage-plugin-checkmarx-backend + title: '@backstage-community/plugin-checkmarx-backend' +spec: + lifecycle: experimental + type: backstage-backend-plugin + owner: maintainers diff --git a/specs/010-catalog-backstage/corpus/workspaces/cicd-statistics/plugins/cicd-statistics-module-buildkite/catalog-info.yaml b/specs/010-catalog-backstage/corpus/workspaces/cicd-statistics/plugins/cicd-statistics-module-buildkite/catalog-info.yaml new file mode 100644 index 00000000..7fa8569d --- /dev/null +++ b/specs/010-catalog-backstage/corpus/workspaces/cicd-statistics/plugins/cicd-statistics-module-buildkite/catalog-info.yaml @@ -0,0 +1,10 @@ +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: backstage-plugin-cicd-statistics-module-buildkite + title: '@backstage-community/plugin-cicd-statistics-module-buildkite' + description: CI/CD Statistics plugin module; Buildkite CI/CD +spec: + lifecycle: experimental + type: backstage-frontend-plugin-module + owner: maintainers diff --git a/specs/010-catalog-backstage/corpus/workspaces/cloudbuild/plugins/cloudbuild/catalog-info.yaml b/specs/010-catalog-backstage/corpus/workspaces/cloudbuild/plugins/cloudbuild/catalog-info.yaml new file mode 100644 index 00000000..2b7abead --- /dev/null +++ b/specs/010-catalog-backstage/corpus/workspaces/cloudbuild/plugins/cloudbuild/catalog-info.yaml @@ -0,0 +1,10 @@ +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: backstage-plugin-cloudbuild + title: '@backstage-community/plugin-cloudbuild' + description: A Backstage plugin that integrates towards Google Cloud Build +spec: + lifecycle: experimental + type: backstage-frontend-plugin + owner: maintainers diff --git a/specs/010-catalog-backstage/corpus/workspaces/code-climate/plugins/code-climate/catalog-info.yaml b/specs/010-catalog-backstage/corpus/workspaces/code-climate/plugins/code-climate/catalog-info.yaml new file mode 100644 index 00000000..74aaf8ce --- /dev/null +++ b/specs/010-catalog-backstage/corpus/workspaces/code-climate/plugins/code-climate/catalog-info.yaml @@ -0,0 +1,9 @@ +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: backstage-plugin-code-climate + title: '@backstage-community/plugin-code-climate' +spec: + lifecycle: experimental + type: backstage-frontend-plugin + owner: maintainers diff --git a/specs/010-catalog-backstage/corpus/workspaces/code-coverage/plugins/code-coverage-backend/catalog-info.yaml b/specs/010-catalog-backstage/corpus/workspaces/code-coverage/plugins/code-coverage-backend/catalog-info.yaml new file mode 100644 index 00000000..81daf642 --- /dev/null +++ b/specs/010-catalog-backstage/corpus/workspaces/code-coverage/plugins/code-coverage-backend/catalog-info.yaml @@ -0,0 +1,10 @@ +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: backstage-plugin-code-coverage-backend + title: '@backstage-community/plugin-code-coverage-backend' + description: A Backstage backend plugin that helps you keep track of your code coverage +spec: + lifecycle: experimental + type: backstage-backend-plugin + owner: alde diff --git a/specs/010-catalog-backstage/corpus/workspaces/codescene/plugins/codescene/catalog-info.yaml b/specs/010-catalog-backstage/corpus/workspaces/codescene/plugins/codescene/catalog-info.yaml new file mode 100644 index 00000000..3003bb06 --- /dev/null +++ b/specs/010-catalog-backstage/corpus/workspaces/codescene/plugins/codescene/catalog-info.yaml @@ -0,0 +1,9 @@ +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: backstage-plugin-codescene + title: '@backstage-community/plugin-codescene' +spec: + lifecycle: experimental + type: backstage-frontend-plugin + owner: maintainers diff --git a/specs/010-catalog-backstage/corpus/workspaces/copilot/plugins/copilot-backend/catalog-info.yaml b/specs/010-catalog-backstage/corpus/workspaces/copilot/plugins/copilot-backend/catalog-info.yaml new file mode 100644 index 00000000..d77ecc34 --- /dev/null +++ b/specs/010-catalog-backstage/corpus/workspaces/copilot/plugins/copilot-backend/catalog-info.yaml @@ -0,0 +1,9 @@ +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: backstage-plugin-copilot-backend + title: '@backstage-community/plugin-copilot-backend' +spec: + lifecycle: experimental + type: backstage-backend-plugin + owner: maintainers diff --git a/specs/010-catalog-backstage/corpus/workspaces/cost-insights/plugins/cost-insights-common/catalog-info.yaml b/specs/010-catalog-backstage/corpus/workspaces/cost-insights/plugins/cost-insights-common/catalog-info.yaml new file mode 100644 index 00000000..ab72402a --- /dev/null +++ b/specs/010-catalog-backstage/corpus/workspaces/cost-insights/plugins/cost-insights-common/catalog-info.yaml @@ -0,0 +1,10 @@ +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: backstage-plugin-cost-insights-common + title: '@backstage-community/plugin-cost-insights-common' + description: Common functionalities for the cost-insights plugin +spec: + lifecycle: experimental + type: backstage-common-library + owner: silver-lining diff --git a/specs/010-catalog-backstage/evidence/README.md b/specs/010-catalog-backstage/evidence/README.md index fda71c80..2f7d39e5 100644 --- a/specs/010-catalog-backstage/evidence/README.md +++ b/specs/010-catalog-backstage/evidence/README.md @@ -31,8 +31,9 @@ mechanism 2 together with that Condition of Acceptance. | `frozen-expectations/audit-record.json` | The independent audit of the oracle | T019 (**not** T014–T018) | | `accept-corpus-freeze/` | The clause-5 gate artifact — `AcceptCorpusFreeze` (`../data-model.md` §17) | T014–T016, T018 | | `accept-corpus-freeze/adequacy-audit.json` | The independent adequacy finding | T019 (**not** T014–T018) | -| `negative-cases/` | Retained failing inputs, per ADR-0016 | T020, T021, T023 | +| `negative-cases/` | Retained failing inputs, per ADR-0016 | T020, T021, T023, T089 | | `barrier-b-checkpoint.json` | The `BARRIER_B_CLEARED` record | T024 | +| `comparison/` | ADR-0020 clause 5 **step (b)** — the post-output comparison | T087–T092 (**Phase F**, after the barrier cleared) | ## 3. The content-hash rule @@ -85,6 +86,37 @@ confirmed by T024, this tree contains and must continue to contain: for any descriptor-sourced entity — not persisted, not in memory, not asserted in a test (`research.md` R4). +### 4.1 What §4 binds, now that Phase F has deposited (added by T087–T092) + +**The three absences above are statements about the freeze**, and they still hold +of `frozen-expectations/` and `accept-corpus-freeze/` exactly as written. Nothing +in Phase F wrote to either; `comparison/expectations-unchanged.json` compares each +frozen hash, recomputed after all of Phase E and Phase F, against the value T024 +recorded before any generator existed, and `scripts/check-freeze-hashes.ts` fails +the build on drift. + +They were also, at the time they were written, true of this whole directory, +because Barrier B had not yet cleared. That is no longer the case, and saying so +is better than leaving a claim that quietly stopped being true: + +- `comparison/` **does** hold generator output — `diff-report.json` records derived + ownership for all 24 frozen entities, and `step-b-record.json` records the + envelope's digest. That is the point of step (b). It could not exist before + T024, and it is written only by tasks that list T024 in `Depends`. +- `comparison/` **is** where the harness's provenance record lives, and the + harness itself lives in `scripts/`, not here. The third bullet above anticipated + this: "the harness … is authored in Phase F (T087), strictly after this freeze + and its audit." +- **No `InputManifest` is committed anywhere**, in this tree or outside it, and + mechanism 1 is unchanged. The comparison harness builds one in a temporary + directory at run time and deletes it; the vendored corpus it reads lives at + `../corpus/`, outside this tree, and is pristine upstream bytes with no + annotation of its own. + +The ordering that matters is preserved and remains checkable: everything under +`frozen-expectations/` and `accept-corpus-freeze/` predates every byte under +`comparison/`, and their hashes are the ones Phase B recorded. + ## 5. Standing honesty constraints on everything in this tree 1. **ADR-0014 rung 1 only.** Nothing here is reference-verified (rung 2) or diff --git a/specs/010-catalog-backstage/evidence/comparison/README.md b/specs/010-catalog-backstage/evidence/comparison/README.md new file mode 100644 index 00000000..0631fc81 --- /dev/null +++ b/specs/010-catalog-backstage/evidence/comparison/README.md @@ -0,0 +1,55 @@ +# `comparison/` — ADR-0020 clause 5 **step (b)** + +Written by Phase F (T087–T092). Step (a) — the pre-output freeze and its +independent audit — lives in `../frozen-expectations/` and +`../accept-corpus-freeze/` and is **not written to by anything here**. + +| File | Written by | Purpose | +| --- | --- | --- | +| `harness-provenance.md` | T087 | That the comparison harness was authored **after** the freeze and its audit — R5 mechanism 3 | +| `diff-report.json` | T088 | The diff itself: derived ownership vs. the frozen expectations, at zero FP / zero FN | +| `step-b-record.json` | T090 | Step (b)'s **own** hashes and **own** PASS/FAIL, inheriting nothing from step (a) | +| `expectations-unchanged.json` | T091 | That the frozen hashes are unchanged from their Phase B values, across all of E and F | +| `reporting-honesty.md` | T092 | What this phase's artifacts do and do not claim | + +Every JSON file here is **produced by a run**, never hand-written: + +```bash +bun run scripts/compare-accept-corpus.ts +``` + +## The two things a reader should check first + +1. **The comparison never wrote to the frozen trees.** `expectations-unchanged.json` + compares each frozen artifact's hash, recomputed now, against the value the + **Barrier B checkpoint** recorded before any generator existed — + `../barrier-b-checkpoint.json`, written by the independent auditor session + rather than by the freeze's author. `scripts/check-freeze-hashes.ts` + independently fails the build on any drift, and has been observed genuinely + failing on a one-byte mutation (`../negative-cases/freeze-drift/`). + + A comparison that passes because the expectations moved is not a passing + comparison. That is the failure this whole apparatus exists to prevent. + +2. **The gate has been observed failing.** `../negative-cases/comparison-mismatch/` + holds the verbatim FAIL output from a deliberately mutated comparison input, + and the restored PASS. A gate seen only green is not coverage (ADR-0016). + +## What step (b) establishes, and what it does not + +It establishes that the generator's derived ownership over this frozen corpus +agrees, exactly and in order, with a **maintainer-authored** expectation set +frozen before any generator output existed. + +It does **not** establish correctness in any absolute sense. The expectations are +the maintainer's own, hand-derived from frozen contracts; agreement between our +implementation and our specification is not independent evidence that either is +right. The envelope's self-digest establishes **integrity**, not correctness — a +semantically wrong envelope can carry a perfectly valid self-digest. + +`reporting-honesty.md` states this in full, and +`scripts/compare-accept-corpus.test.ts` scans every artifact in this directory to +check that none of them says otherwise. + +ADR-0014 **rung 1 only**. Nothing here is reference-verified (rung 2) or +externally validated (rung 3), and nothing here schedules or prepares a release. diff --git a/specs/010-catalog-backstage/evidence/comparison/diff-report.json b/specs/010-catalog-backstage/evidence/comparison/diff-report.json new file mode 100644 index 00000000..432539d5 --- /dev/null +++ b/specs/010-catalog-backstage/evidence/comparison/diff-report.json @@ -0,0 +1,300 @@ +{ + "//": "T088 — ADR-0020 clause 5 step (b): the post-output comparison. Derived ownership for every annotated entity in the frozen accept corpus, diffed against the frozen expectations at zero false positives and zero false negatives. Written by scripts/compare-accept-corpus.ts. The expectations were NOT amended to fit this output, and could not have been: scripts/check-freeze-hashes.ts fails the build on any drift in the frozen trees.", + "task": "T088", + "barrierSide": "BEHIND", + "discharges": [ + "FR-056", + "SC-011" + ], + "verdict": "PASS", + "corpusRef": { + "repository": "github.com/backstage/community-plugins", + "commit": "92e9e4e09c76cc57f3475029b73e5ec84498a459" + }, + "frozenInputs": { + "accept-corpus-freeze/accept-corpus-freeze.json": "f98e6d464b53ba334298c1e5b76bbd0222ff2a460f7637439f0e8d54c9294aca", + "frozen-expectations/frozen-expectation-set.json": "e641ae5e4201a099e92e98fbaa7683bc0eb0290adb01f93abbd474c695c2430c", + "note": "Read, cross-checked against each other, and never written. The freeze itself states that the two artifacts are two views of one freeze and that a divergence between them is a freeze failure; this harness refuses to run if they disagree." + }, + "counts": { + "expectedEntities": 24, + "envelopeEntities": 25, + "entitiesOutsideTheExpectationSet": 1, + "falsePositives": 0, + "falseNegatives": 0, + "otherMismatches": 0, + "countingNote": "Descriptor FILE count and entity DOCUMENT count are different numbers. 24 files are named by the manifest; the envelope carries one entity per admissible YAML document in them, which is more. An entity outside the expectation set is a finding only if it derived a non-empty path set." + }, + "findings": { + "falsePositives": [], + "falseNegatives": [], + "otherMismatches": [] + }, + "entitiesOutsideTheExpectationSet": [ + { + "canonicalId": "component:default/backstage-community-acr-frontend", + "ownershipState": "annotation-absent", + "derivedPaths": [], + "sourcePath": "workspaces/acr/plugins/acr/catalog-info.yaml", + "documentIndexInFile": 1 + } + ], + "derivedOwnership": [ + { + "canonicalId": "component:default/backstage-community-acr", + "ownershipState": "explicit-paths", + "derivedPaths": [ + "workspaces/alpha-tools/**", + "workspaces/alpha.config/**", + "workspaces/alpha/src/**" + ], + "sourcePath": "workspaces/acr/plugins/acr/catalog-info.yaml", + "documentIndexInFile": 0 + }, + { + "canonicalId": "component:default/backstage-plugin-adr-backend", + "ownershipState": "explicit-paths", + "derivedPaths": [ + "packages/cli/**", + "packages/core/**" + ], + "sourcePath": "workspaces/adr/plugins/adr-backend/catalog-info.yaml", + "documentIndexInFile": 0 + }, + { + "canonicalId": "component:default/backstage-plugin-agent-forge", + "ownershipState": "explicit-paths", + "derivedPaths": [ + "**", + ".github/**" + ], + "sourcePath": "workspaces/agent-forge/plugins/agent-forge/catalog-info.yaml", + "documentIndexInFile": 0 + }, + { + "canonicalId": "component:default/backstage-plugin-airbrake-backend", + "ownershipState": "explicit-paths", + "derivedPaths": [ + "docs/**" + ], + "sourcePath": "workspaces/airbrake/plugins/airbrake-backend/catalog-info.yaml", + "documentIndexInFile": 0 + }, + { + "canonicalId": "component:default/backstage-plugin-allure", + "ownershipState": "explicit-paths", + "derivedPaths": [ + "src/README.md", + "src/Utils/**", + "src/utils/**" + ], + "sourcePath": "workspaces/allure/plugins/allure/catalog-info.yaml", + "documentIndexInFile": 0 + }, + { + "canonicalId": "component:default/backstage-plugin-analytics-module-ga4", + "ownershipState": "explicit-paths", + "derivedPaths": [ + "scripts/*.test.ts", + "scripts/build-?.ts", + "scripts/build.ts" + ], + "sourcePath": "workspaces/analytics/plugins/analytics-module-ga4/catalog-info.yaml", + "documentIndexInFile": 0 + }, + { + "canonicalId": "component:default/backstage-plugin-apache-airflow", + "ownershipState": "explicit-paths", + "derivedPaths": [ + "packages/core/**", + "packages/core/src/**" + ], + "sourcePath": "workspaces/apache-airflow/plugins/apache-airflow/catalog-info.yaml", + "documentIndexInFile": 0 + }, + { + "canonicalId": "component:default/backstage-plugin-apollo-explorer", + "ownershipState": "explicit-paths", + "derivedPaths": [ + "plugins/v10/**", + "plugins/v2/**", + "plugins/v_next/**" + ], + "sourcePath": "workspaces/apollo-explorer/plugins/apollo-explorer/catalog-info.yaml", + "documentIndexInFile": 0 + }, + { + "canonicalId": "component:default/backstage-plugin-azure-devops-common", + "ownershipState": "explicit-paths", + "derivedPaths": [ + ".github/**", + ".github/workflows/**", + "Makefile" + ], + "sourcePath": "workspaces/azure-devops/plugins/azure-devops-common/catalog-info.yaml", + "documentIndexInFile": 0 + }, + { + "canonicalId": "component:default/backstage-plugin-azure-sites-backend", + "ownershipState": "explicit-paths", + "derivedPaths": [ + "docs/**", + "examples/**" + ], + "sourcePath": "workspaces/azure-sites/plugins/azure-sites-backend/catalog-info.yaml", + "documentIndexInFile": 0 + }, + { + "canonicalId": "component:default/backstage-plugin-badges-backend", + "ownershipState": "explicit-paths", + "derivedPaths": [ + "a-b/**", + "a.b/**", + "a/**", + "a/b/**" + ], + "sourcePath": "workspaces/badges/plugins/badges-backend/catalog-info.yaml", + "documentIndexInFile": 0 + }, + { + "canonicalId": "component:default/backstage-plugin-bazaar-backend", + "ownershipState": "explicit-paths", + "derivedPaths": [ + "workspaces/alpha-tools/**", + "workspaces/alpha.config/**", + "workspaces/alpha/src/**" + ], + "sourcePath": "workspaces/bazaar/plugins/bazaar-backend/catalog-info.yaml", + "documentIndexInFile": 0 + }, + { + "canonicalId": "component:default/backstage-plugin-bitrise", + "ownershipState": "explicit-paths", + "derivedPaths": [ + "**", + ".github/**" + ], + "sourcePath": "workspaces/bitrise/plugins/bitrise/catalog-info.yaml", + "documentIndexInFile": 0 + }, + { + "canonicalId": "component:default/backstage-plugin-catalog-backend-module-codeowners", + "ownershipState": "explicit-paths", + "derivedPaths": [ + "src/README.md", + "src/Utils/**", + "src/utils/**" + ], + "sourcePath": "workspaces/catalog/plugins/catalog-backend-module-codeowners/catalog-info.yaml", + "documentIndexInFile": 0 + }, + { + "canonicalId": "component:default/backstage-plugin-checkmarx-backend", + "ownershipState": "explicit-paths", + "derivedPaths": [ + "scripts/*.test.ts", + "scripts/build-?.ts", + "scripts/build.ts" + ], + "sourcePath": "workspaces/checkmarx/plugins/checkmarx-backend/catalog-info.yaml", + "documentIndexInFile": 0 + }, + { + "canonicalId": "component:default/backstage-plugin-cicd-statistics-module-buildkite", + "ownershipState": "explicit-paths", + "derivedPaths": [ + "packages/core/**", + "packages/core/src/**" + ], + "sourcePath": "workspaces/cicd-statistics/plugins/cicd-statistics-module-buildkite/catalog-info.yaml", + "documentIndexInFile": 0 + }, + { + "canonicalId": "component:default/backstage-plugin-cloudbuild", + "ownershipState": "explicit-paths", + "derivedPaths": [ + "plugins/v10/**", + "plugins/v2/**", + "plugins/v_next/**" + ], + "sourcePath": "workspaces/cloudbuild/plugins/cloudbuild/catalog-info.yaml", + "documentIndexInFile": 0 + }, + { + "canonicalId": "component:default/backstage-plugin-code-climate", + "ownershipState": "explicit-paths", + "derivedPaths": [ + ".github/**", + ".github/workflows/**", + "Makefile" + ], + "sourcePath": "workspaces/code-climate/plugins/code-climate/catalog-info.yaml", + "documentIndexInFile": 0 + }, + { + "canonicalId": "component:default/backstage-plugin-code-coverage-backend", + "ownershipState": "explicit-paths", + "derivedPaths": [ + "docs/**", + "examples/**" + ], + "sourcePath": "workspaces/code-coverage/plugins/code-coverage-backend/catalog-info.yaml", + "documentIndexInFile": 0 + }, + { + "canonicalId": "component:default/backstage-plugin-codescene", + "ownershipState": "explicit-paths", + "derivedPaths": [ + "a-b/**", + "a.b/**", + "a/**", + "a/b/**" + ], + "sourcePath": "workspaces/codescene/plugins/codescene/catalog-info.yaml", + "documentIndexInFile": 0 + }, + { + "canonicalId": "component:default/backstage-plugin-copilot-backend", + "ownershipState": "explicit-empty", + "derivedPaths": [], + "sourcePath": "workspaces/copilot/plugins/copilot-backend/catalog-info.yaml", + "documentIndexInFile": 0 + }, + { + "canonicalId": "component:default/backstage-plugin-cost-insights-common", + "ownershipState": "annotation-absent", + "derivedPaths": [], + "sourcePath": "workspaces/cost-insights/plugins/cost-insights-common/catalog-info.yaml", + "documentIndexInFile": 0 + }, + { + "canonicalId": "component:default/bitbucket-pull-requests", + "ownershipState": "explicit-paths", + "derivedPaths": [ + "packages/cli/**", + "packages/core/**" + ], + "sourcePath": "workspaces/bitbucket-pull-requests/catalog-info.yaml", + "documentIndexInFile": 0 + }, + { + "canonicalId": "component:default/example-website", + "ownershipState": "explicit-paths", + "derivedPaths": [ + "docs/**" + ], + "sourcePath": "workspaces/bookmarks/plugins/bookmarks/examples/component/catalog-info.yaml", + "documentIndexInFile": 0 + } + ], + "whatThisDoesAndDoesNotEstablish": { + "establishes": "That the generator’s derived ownership over this frozen corpus agrees, exactly and in order, with a maintainer-authored expectation set frozen before any generator output existed.", + "doesNotEstablish": [ + "It does not establish correctness in any absolute sense. The expectations are the maintainer’s own, hand-derived from frozen contracts; agreement between our implementation and our specification is not independent evidence that either is right.", + "The envelope’s self-digest establishes INTEGRITY, not correctness. A semantically wrong envelope can carry a perfectly valid self-digest (ADR-0020 clause 5, FR-058, SC-012).", + "It says nothing about Backstage as a running system. The admissibility warrant is exactly what the four pinned validator predicates return at Backstage commit 1121a4facd9e321179d0402c3f355e4a649e84d9.", + "It does not evidence that the mapping reflects anyone’s actual ownership, that anyone else wants the annotation, or that adoption risk has fallen.", + "ADR-0014 rung 1 only — not reference-verified, not externally validated, and no release is scheduled, prepared, or implied." + ] + } +} diff --git a/specs/010-catalog-backstage/evidence/comparison/expectations-unchanged.json b/specs/010-catalog-backstage/evidence/comparison/expectations-unchanged.json new file mode 100644 index 00000000..1e8850f0 --- /dev/null +++ b/specs/010-catalog-backstage/evidence/comparison/expectations-unchanged.json @@ -0,0 +1,33 @@ +{ + "//": "T091 — the clause-5 prohibition, enforced rather than asserted: the frozen expectations were never amended to fit the output. Phase B’s hash values are read from barrier-b-checkpoint.json (written by the independent auditor session, not by the freeze’s author) and compared against values recomputed from the live artifacts now, after all of Phase E and Phase F.", + "task": "T091", + "barrierSide": "BEHIND", + "discharges": [], + "enforces": "ADR-0020 clause 5 — \"the expectations are never amended to fit the output\"", + "phaseBSource": "specs/010-catalog-backstage/evidence/barrier-b-checkpoint.json → confirmations.mechanism2_hashMatch.recordedHashes", + "alsoRecordedAt": "accept-corpus-freeze/selection-basis.md §8.4 carries the same two values in a table, independently written.", + "comparisons": [ + { + "artifact": "frozen-expectations/frozen-expectation-set.json", + "phaseBRecordedHash": "e641ae5e4201a099e92e98fbaa7683bc0eb0290adb01f93abbd474c695c2430c", + "recomputedNow": "e641ae5e4201a099e92e98fbaa7683bc0eb0290adb01f93abbd474c695c2430c", + "recordedInArtifactNow": "e641ae5e4201a099e92e98fbaa7683bc0eb0290adb01f93abbd474c695c2430c", + "unchangedSincePhaseB": true, + "selfConsistentNow": true + }, + { + "artifact": "accept-corpus-freeze/accept-corpus-freeze.json", + "phaseBRecordedHash": "f98e6d464b53ba334298c1e5b76bbd0222ff2a460f7637439f0e8d54c9294aca", + "recomputedNow": "f98e6d464b53ba334298c1e5b76bbd0222ff2a460f7637439f0e8d54c9294aca", + "recordedInArtifactNow": "f98e6d464b53ba334298c1e5b76bbd0222ff2a460f7637439f0e8d54c9294aca", + "unchangedSincePhaseB": true, + "selfConsistentNow": true + } + ], + "allUnchanged": true, + "whyThisIsNotCircular": "The recomputation derives each hash from the artifact’s bytes; the baseline comes from a record written before any generator existed. An artifact edited to fit output would recompute to a different value than the one Phase B recorded, whatever its own contentHash field says. scripts/check-freeze-hashes.ts catches the narrower case where the bytes moved but the recorded hash did not, and has been observed genuinely failing on a one-byte mutation (negative-cases/freeze-drift/).", + "honesty": { + "rung": "ADR-0014 rung 1 only.", + "scope": "This shows the expectations did not move. It does not show they were right — that is not a property any hash can carry." + } +} diff --git a/specs/010-catalog-backstage/evidence/comparison/harness-provenance.md b/specs/010-catalog-backstage/evidence/comparison/harness-provenance.md new file mode 100644 index 00000000..8268b188 --- /dev/null +++ b/specs/010-catalog-backstage/evidence/comparison/harness-provenance.md @@ -0,0 +1,99 @@ +# T087 — harness provenance: authored **after** the freeze and its audit + +**R5 mechanism 3, and this file is the only artifact that carries it.** + +`../../plan.md` names three enforcement mechanisms for Barrier B, all required, +none sufficient alone. The third is an **ordering** requirement: + +> **Ordering of the comparison harness.** The harness that reads both generator +> output *and* the frozen expectations is written **after** the freeze and its +> audit — never before. ADR-0020 clause 5 requires two distinct steps, each +> recording its own hashes and its own PASS/FAIL. A harness authored first +> collapses them into one step, and the second step's PASS then inherits from the +> first instead of standing alone. + +`../../tasks.md` states the same rule as an anti-verdict: **Phase F may not be +started early "so it is ready."** Authoring it early is the specific harm +mechanism 3 exists to prevent. + +## 1. What was confirmed, and by whom + +`scripts/compare-accept-corpus.ts` and `scripts/compare-accept-corpus.test.ts` +**did not exist prior to T024's confirmation, and did not exist prior to Phase E +producing output.** + +The T024 confirmation is not this session's word for it. It was recorded by the +independent auditor session — the author of the T019 audit procedure and the +R5 mechanism-2 drift check, and *not* the author of the T014–T018 freeze — in +`../barrier-b-checkpoint.json`, under `confirmations.mechanism3_ordering`: + +> **claim**: "No comparison harness exists anywhere in the repository, in any +> branch of this worktree, or in any scratch location." + +Its recorded evidence enumerates where it looked: this worktree tracked and +untracked, `origin/main`, every local and remote branch, and scratch locations +(session-state `files/`, `/tmp`, `.test-output`). It further disposes of the two +near-misses a reader would otherwise have to check by hand — +`packages/adapters/spec-kit/test/harness.ts` (feature 003's sandbox scaffold) and +`packages/evaluator/src/compare.ts` (the generic code-unit comparator primitive) — +neither of which runs a generator or reads the oracle. + +That checkpoint was committed in `b9be3dc` (2026-08-05). + +## 2. What the ordering looks like in git + +| Landmark | Commit | When | +| --- | --- | --- | +| Barrier B — freeze, independent audit, T024 checkpoint | `b9be3dc` | 2026-08-05 | +| Phase E — the assembled generator, which first produced output | `f7384ff` | 2026-08-05 | +| Phase F — this harness | the commit that adds this file | after both | + +`git log --diff-filter=A -- scripts/compare-accept-corpus.ts` names exactly one +commit, and it is later than both landmarks above. Before that commit the path +did not exist on any branch — which is the claim `mechanism3_ordering` had +already recorded, checked again here at authoring time. + +## 3. Why the record is prose rather than a passing test + +A test cannot demonstrate that a file was written late. It can only observe the +present, and the present is exactly the state a harness written too early would +also produce. What is checkable — and is checked, by +`scripts/compare-accept-corpus.test.ts` — is that this record exists, that it +cites `barrier-b-checkpoint.json`, and that the checkpoint really does carry the +`mechanism3_ordering` claim quoted above. The ordering itself rests on that +independently-authored record plus git history, and is stated here rather than +implied. + +## 4. What the harness may and may not do + +The harness **reads** the frozen artifacts and **never writes** to them. Two +independent controls stand behind that, neither of which trusts this sentence: + +- `scripts/check-freeze-hashes.ts` re-derives the canonical content hash of every + frozen artifact on every build and fails on drift. It has been observed + genuinely failing on a one-byte mutation (`../negative-cases/freeze-drift/`). +- `expectations-unchanged.json` (T091) compares each frozen hash, recomputed now, + against the value Phase B recorded before any generator existed. + +**The expectations are never amended to fit the output.** If the two disagree, +either the output is wrong or the expectation is wrong, and the answer is to +report it. A mismatch is a finding, not an obstacle. + +## 5. What a PASS here means + +It means the generator's derived ownership over the frozen accept corpus agrees, +exactly and in order, with a **maintainer-authored** expectation set frozen before +any generator output existed. + +It does not establish correctness. The expectations are the maintainer's +own; agreement between our implementation and our specification is not +independent evidence that either is right. A populated, digest-verified envelope +establishes **integrity** — the bytes were not corrupted or naively mutated — and +says nothing about whether the derived ownership is semantically right. Integrity +and correctness are different properties, and only the first is what a digest +carries. See `reporting-honesty.md`. + +ADR-0014 **rung 1** only. This is maintainer-owned verification, which per +ADR-0014's honesty rules must not be described as external, third-party, or +community adoption; only the corpus *data* is third-party, never the validation. +No release is scheduled, prepared, or implied. diff --git a/specs/010-catalog-backstage/evidence/comparison/reporting-honesty.md b/specs/010-catalog-backstage/evidence/comparison/reporting-honesty.md new file mode 100644 index 00000000..05fd0d89 --- /dev/null +++ b/specs/010-catalog-backstage/evidence/comparison/reporting-honesty.md @@ -0,0 +1,94 @@ +# T092 — reporting honesty: what step (b) claims, and what it does not + +**FR-058, FR-063, SC-012.** ADR-0020 clause 5 is blunt about the distinction this +file exists to protect: + +> A populated, digest-verified envelope proves integrity, not correctness — a +> semantically wrong envelope can carry a perfectly valid self-digest. + +That single sentence is the reason step (b) exists at all. If a valid digest were +evidence that the derivation were right, freezing the expectations would have been +enough and no comparison would be needed. + +## 1. Three claims, and only the third is about the derivation + +| Claim | What carries it | What it is worth | +| --- | --- | --- | +| The envelope's bytes are intact | `digest` (`step-b-record.json` → `ownHashes.envelopeDigest`) | **Integrity.** It detects accidental corruption and naive mutation. It does not detect an adversary who recomputes it, and it says nothing about the contents' meaning. | +| The frozen expectations did not move | `expectations-unchanged.json`, `scripts/check-freeze-hashes.ts` | **Immutability.** It shows the target was not moved to meet the arrow. It does not show the target was in the right place. | +| Derived ownership matches the frozen expectations | `diff-report.json` | **Agreement with a maintainer-authored expectation set**, at zero false positives and zero false negatives, over one frozen corpus. | + +Only the third speaks to the derivation, and even it is narrower than +"the adapter is correct" — see §2. + +## 2. Why agreement is not correctness + +The expectations were written by hand, by the maintainer, by applying frozen +contracts to a maintainer-authored overlay +(`../accept-corpus-freeze/expected-paths.json` → `derivation.howProduced`). The +generator was written by the maintainer too. + +So a PASS says: **our implementation agrees with our specification of it.** That +is worth having — it is exactly what spike 009's oracle never demonstrated, on its +own evidence index's admission that it was "not an executed test harness" — but it +is not independent evidence that either the implementation or the specification is +right. Both could be wrong in the same direction, and this comparison would still +pass. + +What would speak to that is review by someone who authored neither, or use by +someone who is not us. Neither has happened. ADR-0014 **rung 1**. + +## 3. What is deliberately not claimed + +- **Nothing about Backstage as a running system.** The only warrant available is + what four pure validator predicates return when invoked at Backstage commit + `1121a4facd9e321179d0402c3f355e4a649e84d9`. No statement here describes the + behaviour of a Backstage catalog, ingester, or processor. +- **Nothing about adoption.** Zero descriptors in the pinned corpus carry + `adrkit.io/owned-paths`; every annotation in the corpus is the maintainer's own + overlay, and every entity in the envelope carries + `provenance: "maintainer-overlay"` accordingly. Per `../../data-model.md` §10, an + entity asserts third-party adoption only when its ownership state is + `explicit-paths` or `explicit-empty` **and** its provenance is + `upstream-authored`. No entity in this corpus does. Adoption by anyone other than + the maintainer is neither established nor gated by this feature (FR-063). +- **Nothing about scale.** The corpus size is 24 because that is what was frozen. + Per ADR-0012 and FR-055 no minimum entity count is invented, and nothing here + ratifies a production limit. +- **Nothing about rung 2 or rung 3.** This is maintainer-owned verification. Per + ADR-0014's honesty rules it must not be described as external, third-party, or + community adoption; only the corpus **data** is third-party, never the + validation. +- **No release.** ADR-0020 clause 9 defers both the release vehicle and the + decision to release at all to a later record. Nothing in Phase F schedules, + prepares, or implies one. + +## 4. Where the third-party boundary is legible + +Two structural choices keep it visible rather than asserted: + +1. **The vendored corpus carries no annotation.** `../../corpus/` holds the 24 + descriptors as pristine upstream bytes, each verified against the content + address the pinned commit fixes. The maintainer-authored overlay lives in + `../accept-corpus-freeze/overlay.json` and is applied at generation time into a + temporary directory that is deleted. A reader can therefore tell by inspection + which bytes are upstream and which are ours — which is precisely what + `../../data-model.md` §10's `provenance` field exists to make possible, and what + pre-merging the two would have destroyed. +2. **`provenance` is declared, not inferred.** A file that carries the annotation + because an upstream author wrote it and one that carries it because we overlaid + it are byte-identical on disk. The declaration is required, exhaustive, and has + no default, so an omission cannot silently become a third-party adoption claim. + +## 5. How this is enforced rather than promised + +`scripts/compare-accept-corpus.test.ts` scans every artifact this phase writes — +both scripts, both prose files, all three JSON records, the negative case, and the +vendored corpus's own README and manifest — for affirmative overclaims about +correctness and about ADR-0014 rungs, and drives that scan against a fixture that +must trip it so that a clean result means it looked. It separately requires the +diff report to carry both an `establishes` statement and a `doesNotEstablish` +list, and requires this file and `harness-provenance.md` to carry the +integrity/correctness distinction and the rung 1 disclosure. + +A promise in prose is not a control. The scan is. diff --git a/specs/010-catalog-backstage/evidence/comparison/step-b-record.json b/specs/010-catalog-backstage/evidence/comparison/step-b-record.json new file mode 100644 index 00000000..a0ffa338 --- /dev/null +++ b/specs/010-catalog-backstage/evidence/comparison/step-b-record.json @@ -0,0 +1,94 @@ +{ + "//": "T090 — ADR-0020 clause 5 step (b)’s own record. Step (b) inherits nothing from step (a): it recomputes the frozen hashes rather than trusting them, records hashes of its own that did not exist at step (a), and renders its own verdict. Written by scripts/compare-accept-corpus.ts.", + "task": "T090", + "barrierSide": "BEHIND", + "discharges": [ + "FR-057 (step (b) half)" + ], + "step": "ADR-0020 clause 5 step (b) — post-output comparison", + "verdict": "PASS", + "inheritsFromStepA": false, + "whyItInheritsNothing": "FR-057: \"The pre-output freeze/audit (FR-053, FR-054) and the post-output comparison (FR-056) MUST be recorded as two distinct steps, each recording its own hashes and its own PASS/FAIL. Neither may inherit the other’s verdict.\" Step (a)’s PASS is not evidence for this one, and this one’s PASS is not evidence that step (a) was right — it is evidence that the generator agrees with what step (a) froze.", + "recomputedFrozenHashes": { + "frozen-expectations/frozen-expectation-set.json": { + "recorded": "e641ae5e4201a099e92e98fbaa7683bc0eb0290adb01f93abbd474c695c2430c", + "recomputed": "e641ae5e4201a099e92e98fbaa7683bc0eb0290adb01f93abbd474c695c2430c", + "match": true + }, + "accept-corpus-freeze/accept-corpus-freeze.json": { + "recorded": "f98e6d464b53ba334298c1e5b76bbd0222ff2a460f7637439f0e8d54c9294aca", + "recomputed": "f98e6d464b53ba334298c1e5b76bbd0222ff2a460f7637439f0e8d54c9294aca", + "match": true + } + }, + "recomputationNote": "Recorded values are read from the artifacts; recomputed values are derived with the canonical form of evidence/README.md §3 (contentHash key removed, keys ascending by compareCodeUnits, array order preserved, no insignificant whitespace, UTF-8, no trailing newline). A `match: false` here is a freeze failure and fails this record’s verdict.", + "ownHashes": { + "note": "Hashes that belong to step (b) and did not exist at step (a). The corpus was not vendored, no overlaid input existed, and no envelope existed when the freeze was written.", + "vendoredCorpusSha256": { + "workspaces/acr/plugins/acr/catalog-info.yaml": "220d3b8ea574bccce6d8fcd8297566d53b9f22a3f2e60586a5a60fb003544673", + "workspaces/adr/plugins/adr-backend/catalog-info.yaml": "ef777c111a23d1ab714aeb2407e1403e9bfee84b61694dcc75c30534942cf527", + "workspaces/agent-forge/plugins/agent-forge/catalog-info.yaml": "7b7a57dd12c6e60dde41833037b3bd77f8fb3c8cb7b4a113f3d3c5aeb7540849", + "workspaces/airbrake/plugins/airbrake-backend/catalog-info.yaml": "f12c328e2c2a9e219f631b5a6a2bb179c4883dd3f1f01e2d12d5363afcd2af7f", + "workspaces/allure/plugins/allure/catalog-info.yaml": "70d8db4a983b8afac837e5179c25b032009c3ea08ff9607bf7d1de6d3f432afe", + "workspaces/analytics/plugins/analytics-module-ga4/catalog-info.yaml": "117b126e92e30e49dd6444cb441519d1db898021b113755140b3da1d3987a58e", + "workspaces/apache-airflow/plugins/apache-airflow/catalog-info.yaml": "f1bafbb958155dd813c1e10d3a7f8d3ab632c31adf0e535475204ee14a7d7fe6", + "workspaces/apollo-explorer/plugins/apollo-explorer/catalog-info.yaml": "fa5d34aebc5214880e9c89490d5937cfcd48fdc6dae290ce67ade97755576275", + "workspaces/azure-devops/plugins/azure-devops-common/catalog-info.yaml": "ba961f94c5718d2bb9e61c4fa1ad4f22dbe384db99572e2c57d144f7bf3280d7", + "workspaces/azure-sites/plugins/azure-sites-backend/catalog-info.yaml": "847a38e3525f0ed2e6bd291c1b05295043407704f6005414ce9e02a41d333e69", + "workspaces/badges/plugins/badges-backend/catalog-info.yaml": "802effad65014eae49868aa6c58f33a28fd936ff3ee27cce1927a17f814855d0", + "workspaces/bazaar/plugins/bazaar-backend/catalog-info.yaml": "f29955f93d8d86ac590447eb5739d94a46dfd7ffa67c5d7778c55b3f30285871", + "workspaces/bitbucket-pull-requests/catalog-info.yaml": "caf63c13bc985053afd9177c6746e9a771ce85a1411991964af7b35efa9fbb0b", + "workspaces/bitrise/plugins/bitrise/catalog-info.yaml": "3ac10cd165437e2635b6be691385350d592134d4446847b3f177aaaa1cb74a20", + "workspaces/bookmarks/plugins/bookmarks/examples/component/catalog-info.yaml": "d2e57cf42130dd65243d41613dedcaafbd9dfa1dd54f3b31f37046dec426d68e", + "workspaces/catalog/plugins/catalog-backend-module-codeowners/catalog-info.yaml": "1ac0c4ef588570bef861a2da35ddce18b7f66d8d555d4ed6549759ecb2537b01", + "workspaces/checkmarx/plugins/checkmarx-backend/catalog-info.yaml": "9a746e20e8b82b8d5af9bae1c94bb87fe5dc6a32b6867eb5a59863077577b48e", + "workspaces/cicd-statistics/plugins/cicd-statistics-module-buildkite/catalog-info.yaml": "6c6711934fed735785325e1e52838003c76cc6b3268d9f196aa96e9be77b82ca", + "workspaces/cloudbuild/plugins/cloudbuild/catalog-info.yaml": "59049aa4342bdc43e284cd9e67e654ae9221a1223b938c3318d3585a401560a1", + "workspaces/code-climate/plugins/code-climate/catalog-info.yaml": "f20b9ba12a185e6abe23dda6b311c46f1a9378cce2daa1a80a039d51425c7ce4", + "workspaces/code-coverage/plugins/code-coverage-backend/catalog-info.yaml": "eca711137951f49b96a05840c1fceee73128657b13eb3d7536b3734175cc9471", + "workspaces/codescene/plugins/codescene/catalog-info.yaml": "71ccfddf6aa1157746870fbb84ff7dcaff49ebd54e1eda8489bf0c4451312a35", + "workspaces/copilot/plugins/copilot-backend/catalog-info.yaml": "642ed324ea086ac9c56ae952865f9ee65f576b36418b9d3f9bba6eddb2947177", + "workspaces/cost-insights/plugins/cost-insights-common/catalog-info.yaml": "9c98f90b3e03427f6c6dabd4b81125f6e4034c68fea6225e7110c99807a38c5a" + }, + "overlaidSourceDigests": { + "workspaces/acr/plugins/acr/catalog-info.yaml": "7c5fab4b696f05191c7f0fe7146a763420e0001d3a84f06798f7538e109bf119", + "workspaces/adr/plugins/adr-backend/catalog-info.yaml": "e554bf163ef443d82796c4cfe2bc84c3fdbb0bc12d4238ad1a90b7795a40b135", + "workspaces/agent-forge/plugins/agent-forge/catalog-info.yaml": "dfd5953b68413178c38778cd19ed6bf9ebdf81923d40f2cbfff1b0891151e3c9", + "workspaces/airbrake/plugins/airbrake-backend/catalog-info.yaml": "41e083da37cc30236aca2d8997b8752776352c9cd6e5ce8041c61561a250ecab", + "workspaces/allure/plugins/allure/catalog-info.yaml": "d511efbae314c83194a41091e03588c6594392daacd0534931d20cffee7145ba", + "workspaces/analytics/plugins/analytics-module-ga4/catalog-info.yaml": "e68e5acf25bf7b037285ee704d34d76aeadb604cf3844ac682e0b8a06b2d743e", + "workspaces/apache-airflow/plugins/apache-airflow/catalog-info.yaml": "4611848b1432a6c911485e58943727527a6843a1f9a7d9de2a85589b84a876a2", + "workspaces/apollo-explorer/plugins/apollo-explorer/catalog-info.yaml": "7911cf828ad1887ee0bbae555d12ac853bd1626ed3513334d7368a1c11166439", + "workspaces/azure-devops/plugins/azure-devops-common/catalog-info.yaml": "4752c7801b07290d187e40b9b142673e817da94b81cf8a2d562d2f439aec791d", + "workspaces/azure-sites/plugins/azure-sites-backend/catalog-info.yaml": "698e412be4d6307f6333d5426f36ab4ba516e056e1d7a612c1887094e346341d", + "workspaces/badges/plugins/badges-backend/catalog-info.yaml": "35979b164b4e069497e29c9c3b01a0d2590bb1a729827861c8af8cbbcd2c6f75", + "workspaces/bazaar/plugins/bazaar-backend/catalog-info.yaml": "3ad0877eed08d67abf32b590ea6bc493d027a8c339190230dd3a8b4568c086dd", + "workspaces/bitbucket-pull-requests/catalog-info.yaml": "f8410aa3765c6a92eb7c050cd4e93c011bc87e00b43cebaf4a0003052ad59330", + "workspaces/bitrise/plugins/bitrise/catalog-info.yaml": "2848fe749ec27918e23a7209029aa6c27158721f9a0561f8c9e31535f0e578c8", + "workspaces/bookmarks/plugins/bookmarks/examples/component/catalog-info.yaml": "b255ca12e3df990fc934b5e305ddd001fb3cf92e16e793b861392c422761d138", + "workspaces/catalog/plugins/catalog-backend-module-codeowners/catalog-info.yaml": "e688087325945d5e47bb89d5ec1ea3ca63fa85e3d9289636157468c22ab187b9", + "workspaces/checkmarx/plugins/checkmarx-backend/catalog-info.yaml": "ec936633e122754c4cbc095d5c004d2313270c4cddca1c0fc025165e933a6ccd", + "workspaces/cicd-statistics/plugins/cicd-statistics-module-buildkite/catalog-info.yaml": "e915fd8817060d9d9887b3f6c3937ee04247e04aeed4eab2614287d507e8d526", + "workspaces/cloudbuild/plugins/cloudbuild/catalog-info.yaml": "63702bff21e209749d559884c018b1d3060c9eafabf9bc9951bb5f5097ee809b", + "workspaces/code-climate/plugins/code-climate/catalog-info.yaml": "c8d454c4cd4fa5fe5f358e9d13120d25c0242f6cc298a94bb33d955e89508631", + "workspaces/code-coverage/plugins/code-coverage-backend/catalog-info.yaml": "d65ee4d9e0f836504b9be936ea30e97793ba196006d9ef037a753c162f1d7d76", + "workspaces/codescene/plugins/codescene/catalog-info.yaml": "c50ac781577e2ef76317f05f9e1cb35e8ff9e2b148d184744d9f7df55badd8b4", + "workspaces/copilot/plugins/copilot-backend/catalog-info.yaml": "48c3749e49b7960890e14c1f9d0beee877fe29069709be8975cf6b372ac0c998", + "workspaces/cost-insights/plugins/cost-insights-common/catalog-info.yaml": "9c98f90b3e03427f6c6dabd4b81125f6e4034c68fea6225e7110c99807a38c5a" + }, + "envelopeDigest": "45995d938a690aa11d8b139f58d396f3d7eb80fc61f143f23688ff74b1a87836", + "envelopeDigestMeans": "INTEGRITY, not correctness. A semantically wrong envelope can carry a perfectly valid self-digest (ADR-0020 clause 5, FR-058, SC-012). What speaks to the derivation is the comparison, and what the comparison speaks to is agreement with a maintainer-authored expectation set — not correctness in any absolute sense." + }, + "counts": { + "expectedEntities": 24, + "envelopeEntities": 25, + "falsePositives": 0, + "falseNegatives": 0, + "otherMismatches": 0 + }, + "honesty": { + "rung": "ADR-0014 rung 1 only. Maintainer-owned verification, which is not external, third-party, or community adoption. Only the corpus DATA is third-party.", + "release": "No release is scheduled, prepared, or implied. ADR-0020 clause 9 defers both the vehicle and the decision to release at all to a later record.", + "backstage": "No claim is made about Backstage as a running system. The admissibility warrant is exactly what the four pinned validator predicates return at Backstage commit 1121a4facd9e321179d0402c3f355e4a649e84d9." + } +} diff --git a/specs/010-catalog-backstage/evidence/negative-cases/comparison-mismatch/README.md b/specs/010-catalog-backstage/evidence/negative-cases/comparison-mismatch/README.md new file mode 100644 index 00000000..774722cd --- /dev/null +++ b/specs/010-catalog-backstage/evidence/negative-cases/comparison-mismatch/README.md @@ -0,0 +1,90 @@ +# Negative case: comparison mismatch (T089) + +**Retained permanent negative case for the ADR-0016 observation of the ADR-0020 +clause-5 step (b) comparison gate (`scripts/compare-accept-corpus.ts`).** + +The gate requires derived ownership for every annotated entity in the frozen +accept corpus to match the frozen expectations at **zero false positives and zero +false negatives**. A gate that has only ever been seen green is not coverage, so +it was made to fail. + +## What was constructed + +A deliberate mismatch was introduced into the **comparison input** — the corpus +side — and never into the frozen expectations. One overlay annotation value was +replaced with a plausible-looking near-miss: + +| | Value | +| --- | --- | +| Frozen overlay for `workspaces/adr/plugins/adr-backend/catalog-info.yaml[0]` | `["packages/core/**","packages/core/**","packages/cli/**"]` | +| Mutated value | `["packages/core/**","packages/clx/**"]` | + +`cli` → `clx`: one character. The near-miss is the point. A wholesale replacement +would fail on everything at once and would not show that the comparison +discriminates; this yields **exactly one false negative** and **exactly one false +positive** on the same entity, plus the corpus-wide pattern-union mismatch that +follows from it. + +The mutation lives in source as the named constant `T089_MUTATION` in +`scripts/compare-accept-corpus.ts`, so the retained artifact is a line of code +rather than a shell invocation somebody has to reproduce exactly. It is reachable +only through the `--observe-failing` flag, which writes no report; the committed +`diff-report.json` always describes an unmutated run. + +## The command that produced the failure + +```bash +bun run scripts/compare-accept-corpus.ts --observe-failing +``` + +`bun run scripts/compare-accept-corpus.ts` (no flag) is **not** interchangeable +with it: the unflagged form runs the real comparison and passes. + +## Observed output + +Captured verbatim into `observed-fail.txt` by the run itself, not transcribed: + +``` +compare-accept-corpus: FAIL — 24 expected entities, 1 false positive(s), 1 false negative(s), 1 other mismatch(es) + false-negative component:default/backstage-plugin-adr-backend: expected path was not derived — expected "packages/cli/**"; derivedPaths = ["packages/clx/**","packages/core/**"] + false-positive component:default/backstage-plugin-adr-backend: derived path is not in the frozen expectation — derived "packages/clx/**"; expectedPaths = ["packages/cli/**","packages/core/**"] + other-mismatch (corpus-wide): the union of derived patterns does not match the oracle derivedPathPatterns — oracle records 25 patterns, output yields 26 +exit=1 +``` + +Removing the mutation returns the gate to PASS, captured in +`restored.observed.txt`: + +``` +compare-accept-corpus: PASS — 24 expected entities, 0 false positive(s), 0 false negative(s), 0 other mismatch(es) +exit=0 +``` + +## What was not touched + +**The frozen expectations.** Nothing in this cycle wrote to +`../../frozen-expectations/` or `../../accept-corpus-freeze/`, and two independent +controls would have caught it if anything had: `bun run check:freeze-hashes` and +`../../comparison/expectations-unchanged.json`, which compares each frozen hash +recomputed now against the value the Barrier B checkpoint recorded before any +generator existed. + +This directory holds no mutated artifact of its own, for the same reason +`freeze-drift/` does not: a mutated file left in the tree would make a live check +fail on every build. + +## Where the permanent automated case lives + +`scripts/compare-accept-corpus.test.ts` performs the whole mutate → FAIL → +restore → PASS cycle in-suite against the real corpus, asserts the exact reason +strings above, and separately asserts that the frozen artifact's bytes are +identical before and after the mutated run. Every individual failure mode of the +comparison kernel — missing entity, undelivered expected path, unlicensed derived +path, wrong ownership state, right members in the wrong order, wrong source +document, wrong pattern union, and an out-of-set entity deriving ownership — is +additionally driven against an input built to trip it. + +## Standing constraints + +ADR-0014 **rung 1 only**. This observation is maintainer-owned, which is not +external, third-party, or community validation. diff --git a/specs/010-catalog-backstage/evidence/negative-cases/comparison-mismatch/observed-fail.txt b/specs/010-catalog-backstage/evidence/negative-cases/comparison-mismatch/observed-fail.txt new file mode 100644 index 00000000..5ce6efbb --- /dev/null +++ b/specs/010-catalog-backstage/evidence/negative-cases/comparison-mismatch/observed-fail.txt @@ -0,0 +1,6 @@ +compare-accept-corpus: --observe-failing — the comparison INPUT was deliberately mutated at workspaces/adr/plugins/adr-backend/catalog-info.yaml[0]. The frozen expectations were not touched, and no report was written. +compare-accept-corpus: FAIL — 24 expected entities, 1 false positive(s), 1 false negative(s), 1 other mismatch(es) + false-negative component:default/backstage-plugin-adr-backend: expected path was not derived — expected "packages/cli/**"; derivedPaths = ["packages/clx/**","packages/core/**"] + false-positive component:default/backstage-plugin-adr-backend: derived path is not in the frozen expectation — derived "packages/clx/**"; expectedPaths = ["packages/cli/**","packages/core/**"] + other-mismatch (corpus-wide): the union of derived patterns does not match the oracle derivedPathPatterns — oracle records 25 patterns, output yields 26: ["**",".github/**",".github/workflows/**","Makefile","a-b/**","a.b/**","a/**","a/b/**","docs/**","examples/**","packages/cli/**","packages/clx/**","packages/core/**","packages/core/src/**","plugins/v10/**","plugins/v2/**","plugins/v_next/**","scripts/*.test.ts","scripts/build-?.ts","scripts/build.ts","src/README.md","src/Utils/**","src/utils/**","workspaces/alpha-tools/**","workspaces/alpha.config/**","workspaces/alpha/src/**"] +exit=1 diff --git a/specs/010-catalog-backstage/evidence/negative-cases/comparison-mismatch/restored.observed.txt b/specs/010-catalog-backstage/evidence/negative-cases/comparison-mismatch/restored.observed.txt new file mode 100644 index 00000000..89777885 --- /dev/null +++ b/specs/010-catalog-backstage/evidence/negative-cases/comparison-mismatch/restored.observed.txt @@ -0,0 +1,2 @@ +compare-accept-corpus: PASS — 24 expected entities, 0 false positive(s), 0 false negative(s), 0 other mismatch(es) +exit=0 diff --git a/specs/010-catalog-backstage/tasks.md b/specs/010-catalog-backstage/tasks.md index 8eb8bf33..7201deb0 100644 --- a/specs/010-catalog-backstage/tasks.md +++ b/specs/010-catalog-backstage/tasks.md @@ -1084,7 +1084,39 @@ is ready."** Authoring the comparison harness before the freeze and the audit wo collapse ADR-0020 clause 5's two distinct steps into one, which is the exact failure the barrier exists to prevent. -- [ ] T087 [US7] **Author the comparison harness now, and not before.** Write +- [X] T086a [US7] **Vendor the frozen accept corpus.** New work, not in the original + task list, added by maintainer decision 2026-08-05 after Phase E discovered that + the accept corpus is **not materialized in this repository**: the freeze records + corpus *metadata* (paths, canonical ids, overlay values, expected paths), never + descriptor files, and `/README.md` §4 requires the freeze tree to stay + that way because R5 mechanism 1 (input absence) depends on it. T088 cannot run + without the descriptors, and neither can SC-009 limb 2. + Vendor the 24 selected descriptors **verbatim** from + `backstage/community-plugins` at the pinned commit, verify each against the + content address that commit fixes, and **abort on any mismatch** — a mismatch + means the pin moved or the fetch is wrong, and neither is papered over. + **Vendor PRISTINE upstream bytes; keep the overlay separate.** ADR-0020 clause 5 + requires the descriptors be "authored upstream and otherwise unmodified", which is + provable by digest only if what lands on disk is byte-identical to upstream; + `data-model.md` §10's `provenance` exists to keep that boundary legible, and a + pre-merged file destroys it. The overlay stays in + `/accept-corpus-freeze/overlay.json` and is applied by T087's harness at + generation time, into a temporary directory that is deleted. + **Acquisition is not generation.** Fetching is a one-time step whose output is + committed; FR-018 and FR-052 keep generation offline, credential-free, and + network-free, so the acquisition script must not be reachable from the generator + or from the comparison harness. + Files: `scripts/vendor-accept-corpus.ts`, `scripts/vendor-accept-corpus.test.ts`, + `specs/010-catalog-backstage/corpus/**` (24 descriptors, `VENDOR-MANIFEST.json`, + `README.md`). + Barrier: BEHIND — it materializes an input the generator can read, so it may not + precede T024. It is nonetheless **not** generator output under R4: it produces no + ownership result, and its content is fixed by an upstream commit rather than by + anything derived from the frozen expectations. + Discharges: none — unblocks T088, and unblocks SC-009 limb 2 for T086 + Depends: T024, T086 + +- [X] T087 [US7] **Author the comparison harness now, and not before.** Write `scripts/compare-accept-corpus.ts` and `scripts/compare-accept-corpus.test.ts`. Record explicitly, at `/comparison/harness-provenance.md`, that no comparison harness existed prior to T024's confirmation and prior to Phase E @@ -1094,7 +1126,7 @@ the barrier exists to prevent. Discharges: none — implements R5 mechanism 3 Depends: T024, T086 -- [ ] T088 [US7] Diff the derived ownership for **every annotated entity in the frozen +- [X] T088 [US7] Diff the derived ownership for **every annotated entity in the frozen accept corpus** against the frozen expectations, requiring **zero false positives and zero false negatives**. Files: `scripts/compare-accept-corpus.ts`, @@ -1103,7 +1135,7 @@ the barrier exists to prevent. Discharges: FR-056, SC-011 Depends: T024, T087 -- [ ] T089 [US7] **Observed failing.** Introduce a deliberate mismatch into the +- [X] T089 [US7] **Observed failing.** Introduce a deliberate mismatch into the comparison input; observe the gate FAIL and record the exact reason; remove the mismatch; observe the PASS. Retain the mismatch as a permanent negative case at `/negative-cases/comparison-mismatch/`. @@ -1111,14 +1143,14 @@ the barrier exists to prevent. Discharges: none — supplies the ADR-0016 observation for SC-011 Depends: T024, T088 -- [ ] T090 [US7] Record step (b)'s **own** hashes and **own** PASS/FAIL at +- [X] T090 [US7] Record step (b)'s **own** hashes and **own** PASS/FAIL at `/comparison/step-b-record.json`. Step (b) inherits nothing from step (a): it recomputes, and it renders its own verdict. Barrier: BEHIND Discharges: FR-057 (step (b) half) Depends: T024, T089 -- [ ] T091 [US7] **Prohibition guard: expectations are never amended to fit output.** +- [X] T091 [US7] **Prohibition guard: expectations are never amended to fit output.** Assert that every hash under `/frozen-expectations/` and `/accept-corpus-freeze/` is unchanged from its Phase B value, across the whole of Phase E and Phase F. A comparison that passes because the @@ -1129,7 +1161,7 @@ the barrier exists to prevent. Discharges: none — enforces the clause-5 prohibition Depends: T024, T090 -- [ ] T092 [US7] Reporting-honesty close-out: assert that no Phase F artifact presents +- [X] T092 [US7] Reporting-honesty close-out: assert that no Phase F artifact presents the populated, digest-verified envelope as evidence of **correctness**. A digest establishes integrity. The comparison establishes agreement with a maintainer-authored expectation set. Neither establishes that the adapter is correct, and no artifact From 22a452c75753dbfb99a8db2eeb2f7ce262da8aff Mon Sep 17 00:00:00 2001 From: Mark Beacom Date: Wed, 5 Aug 2026 19:39:38 -0400 Subject: [PATCH 2/2] fix(catalog): match the corpus host as a token, not as a URL substring CodeQL flagged `text.includes('https://api.github.com')` under js/incomplete-url-substring-sanitization at high severity. The security concern does not apply here - this scans our own source text to check that exactly one script names the corpus host, and sanitizes nothing - but the rule is right that substring-matching a URL is a poor way to reason about hosts, and the guard had a real weakness because of it. The old check missed `http://api.github.com` and protocol-relative `//api.github.com`. Matching the hostname as a token catches every scheme, so this is the stronger guard rather than a differently-spelled one. The trailing (?![\w.-]) is what keeps it honest: it rejects `api.github.com.evil.com`, which is the exact defect the CodeQL rule exists to name and which a substring test would have accepted. Verified against six cases: the real usage, http, protocol-relative, the attack shape, an unrelated near-miss, and a clean file - all correct. Gates: bun test 1791 pass / 0 fail, typecheck clean, check:freeze-hashes ok, and compare-accept-corpus still PASS at 0 false positives / 0 false negatives. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- scripts/vendor-accept-corpus.test.ts | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/scripts/vendor-accept-corpus.test.ts b/scripts/vendor-accept-corpus.test.ts index d7b49d86..342ea0df 100644 --- a/scripts/vendor-accept-corpus.test.ts +++ b/scripts/vendor-accept-corpus.test.ts @@ -334,13 +334,30 @@ describe('T086a — acquisition is not reachable from generation (FR-018, FR-052 test('exactly one script names the corpus host, and it is the acquisition script', async () => { // A weaker guard than the import checks and stated as such: it catches a second // fetcher being added beside the generator, not every conceivable route to a network. + // + // Matched as a *hostname token* rather than as the substring "https://api.github.com". + // Two reasons, and the second is the one that matters: + // + // 1. A full-URL substring test misses `http://api.github.com` and protocol-relative + // `//api.github.com`. Matching the host itself catches every scheme, so this is + // the stronger check, not merely a differently-spelled one. + // 2. CodeQL flags `text.includes('https://api.github.com')` under + // `js/incomplete-url-substring-sanitization` — the rule against validating an + // untrusted URL by substring, where `https://api.github.com.evil.com` would pass. + // That concern does not apply here (this scans our own source text; it sanitizes + // nothing), but the rule is right that the *pattern* is a poor way to reason about + // URLs, and the token form is both accurate and unambiguous about its intent. + // + // The trailing `(?![\w.-])` is what keeps it honest: it prevents `api.github.com.evil` + // from matching, which is the very defect the CodeQL rule exists to name. + const CORPUS_HOST = /\bapi\.github\.com(?![\w.-])/u; const files = (await sources('scripts')).filter( (file) => !EXCLUDED_FROM_HOST_SCAN.includes(file), ); const fetchers: string[] = []; for (const file of files) { const text = await readFile(join(REPO_ROOT, file), 'utf8'); - if (text.includes('https://api.github.com')) fetchers.push(file); + if (CORPUS_HOST.test(text)) fetchers.push(file); } expect(fetchers).toEqual(['scripts/vendor-accept-corpus.ts']); });