Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions .github/workflows/docs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,19 @@ jobs:
- name: Self-tests for the ADR ruleset generator
run: node --test .harness/scripts/generate-adr-rulesets.test.mjs

# GT-598 / GT-633: `native-evaluability-snapshot.json` says of itself that it
# is a capture, and for a long time nothing captured it — it was written by
# hand and drifted, while its guard compared the file against six numbers
# typed into the test: the same six the file contained. Drift is not
# contained either, because `build-iso-5055-mapping.mjs` stamps
# nativeEvaluability onto all 388 mapping rows FROM this file. The order of
# these two steps is the order of the chain.
- name: Native evaluability snapshot is a faithful capture of Core's triage
run: node src/rulesets/standards/capture-native-evaluability-snapshot.mjs --check

- name: ISO/IEC 5055 mapping guard
run: node --test src/rulesets/standards/iso-5055-mapping.test.mjs

# GT-630: the derived artifacts have a dependency ORDER — the executive
# summary is built FROM the maturity reconciliation, which is built from
# the board. Generate the summary first and it captures a value the
Expand Down
32 changes: 32 additions & 0 deletions .harness/scripts/ci/46-validate-derived-artifact-order.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,38 @@ export const CHAIN = [
consumes: ['src/packages/mcp-server/src/mcp/abac-evaluator.ts'],
writes: ['src/rulesets/opa/abac-mcp-tool-access.rego'],
},
{
name: 'native evaluability snapshot',
producer: 'src/rulesets/standards/capture-native-evaluability-snapshot.mjs',
checkArgs: ['--check'],
// Same shape as the ABAC link: derived from the runtime through ts-node, so
// its real inputs are Core's triage sources plus the corpus they classify.
// The corpus half matters — `documentation-only` moved 129 -> 136 purely
// because seven generated ADR rulesets appeared, and nothing noticed for as
// long as this file was maintained by hand (GT-598).
consumes: [
'src/packages/core-domain/test/rule-corpus-triage.ts',
'src/packages/core-domain/src/application/validators/rule-evaluability.ts',
'src/packages/core-domain/src/application/validators/evaluators/native-evaluator.ts',
],
writes: ['src/rulesets/standards/native-evaluability-snapshot.json'],
},
{
name: 'ISO/IEC 5055 corpus mapping',
producer: 'src/rulesets/standards/build-iso-5055-mapping.mjs',
checkArgs: ['--check'],
// THE edge this pair exists for: the generator stamps `nativeEvaluability`
// onto every row from the snapshot. Rebuild before recapturing and a stale
// class is laundered into a 391-row artifact, with the handler backlog
// overstated by exactly the rules Core has closed since the last capture.
// Both artifacts pass their own --check at that moment, which is precisely
// the failure one-artifact-at-a-time checking cannot see.
consumes: ['src/rulesets/standards/native-evaluability-snapshot.json'],
writes: [
'src/rulesets/standards/iso-5055-mapping.json',
'src/rulesets/standards/iso-5055-mapping.csv',
],
},
{
name: 'maturity reconciliation',
producer: '.harness/scripts/ci/09-reconcile-maturity.mjs',
Expand Down
78 changes: 64 additions & 14 deletions .harness/scripts/ci/46-validate-derived-artifact-order.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,22 @@ import { spawnSync } from 'node:child_process';
const __dirname = dirname(fileURLToPath(import.meta.url));
const GUARD = resolve(__dirname, '46-validate-derived-artifact-order.mjs');

// The guard only runs main() when invoked directly, so importing CHAIN is safe.
const { CHAIN } = await import('./46-validate-derived-artifact-order.mjs');

/**
* Where a link sits, DERIVED rather than typed.
*
* These positions were hardcoded ("link 2 of 3"), so every link added to the
* chain broke assertions that were not about the new link at all — which reads
* as the change being wrong rather than the test being brittle.
*/
const linkPosition = (producerSuffix) => {
const i = CHAIN.findIndex((l) => l.producer.endsWith(producerSuffix));
assert.notEqual(i, -1, `no chain link produces ${producerSuffix} — the fixture is describing a chain that moved`);
return { position: i + 1, total: CHAIN.length };
};

let sandbox;
before(() => { sandbox = mkdtempSync(join(tmpdir(), 'gt630-')); });
after(() => { if (sandbox) rmSync(sandbox, { recursive: true, force: true }); });
Expand All @@ -38,7 +54,7 @@ test('the real repository is current and at a fixed point', () => {
const { status, out } = run(resolve(__dirname, '../../..'));
assert.equal(status, 0, out);
assert.match(out, /at a fixed point/);
assert.match(out, /links declared \.+ 3/);
assert.match(out, /links declared \.+ 5/);
});

test('the guard leaves the real tree byte-identical', () => {
Expand All @@ -50,6 +66,13 @@ test('the guard leaves the real tree byte-identical', () => {
'reference/core/control-center/maturity-reports/maturity-reconciliation.json',
'reference/core/control-center/maturity-reports/executive-summary.md',
'reference/core/control-center/maturity-reports/executive-summary.es.md',
// GT-598's pair. The snapshot is the sharper case: its producer stamps a
// `capturedOn` date, and it is deliberately STICKY when the classification
// is unchanged. Were it not, a replay would rewrite the date, this assertion
// would fail, and the fixed-point pass would report drift every day.
'src/rulesets/standards/native-evaluability-snapshot.json',
'src/rulesets/standards/iso-5055-mapping.json',
'src/rulesets/standards/iso-5055-mapping.csv',
];
const before = artifacts.map((a) => readFileSync(join(repo, a)));
run(repo);
Expand All @@ -60,25 +83,49 @@ test('the guard leaves the real tree byte-identical', () => {


/**
* The chain gained a first link (the ABAC rego, GT-602). These fixtures are about
* ORDER, not about ABAC, so each mini-repo gets a trivial producer/artifact pair
* for it — otherwise the shape check trips before the behaviour under test runs.
* Every REAL chain link that is not the behaviour under test needs a trivial
* producer/artifact pair in each mini-repo, or the shape check trips before the
* test gets to run. These fixtures are about ORDER — not about ABAC, the
* evaluability snapshot, or the 5055 mapping.
*
* Keep this in step with CHAIN. A link added to the guard without a stub here
* fails every fixture below with "declared producer does not exist", which
* points at the fixture rather than at the change that caused it.
*/
const ABAC_STUB = {
'.harness/scripts/generate-abac-tool-sets.mjs':
"import fs from 'node:fs';\nconst f = process.cwd() + '/src/rulesets/opa/abac-mcp-tool-access.rego';\n" +
"if (process.argv.includes('--check')) process.exit(fs.readFileSync(f, 'utf8') === 'stable\\n' ? 0 : 1);\n" +
"fs.writeFileSync(f, 'stable\\n');\n",
const stubProducer = (artifacts) =>
"import fs from 'node:fs';\n" +
`const files = ${JSON.stringify(artifacts)}.map(f => process.cwd() + '/' + f);\n` +
"if (process.argv.includes('--check')) " +
"process.exit(files.every(f => fs.readFileSync(f, 'utf8') === 'stable\\n') ? 0 : 1);\n" +
"for (const f of files) fs.writeFileSync(f, 'stable\\n');\n";

const PRELUDE_STUBS = {
// link 1 — ABAC rego (GT-602)
'.harness/scripts/generate-abac-tool-sets.mjs': stubProducer(['src/rulesets/opa/abac-mcp-tool-access.rego']),
'src/packages/mcp-server/src/mcp/abac-evaluator.ts': '// stub\n',
'src/rulesets/opa/abac-mcp-tool-access.rego': 'stable\n',

// link 2 — native evaluability snapshot (GT-598)
'src/rulesets/standards/capture-native-evaluability-snapshot.mjs':
stubProducer(['src/rulesets/standards/native-evaluability-snapshot.json']),
'src/packages/core-domain/test/rule-corpus-triage.ts': '// stub\n',
'src/packages/core-domain/src/application/validators/rule-evaluability.ts': '// stub\n',
'src/packages/core-domain/src/application/validators/evaluators/native-evaluator.ts': '// stub\n',
'src/rulesets/standards/native-evaluability-snapshot.json': 'stable\n',

// link 3 — ISO/IEC 5055 mapping (GT-598), which consumes link 2's artifact
'src/rulesets/standards/build-iso-5055-mapping.mjs':
stubProducer(['src/rulesets/standards/iso-5055-mapping.json', 'src/rulesets/standards/iso-5055-mapping.csv']),
'src/rulesets/standards/iso-5055-mapping.json': 'stable\n',
'src/rulesets/standards/iso-5055-mapping.csv': 'stable\n',
};

// --- the shape of the declaration itself ------------------------------------

describe('chain declaration (anti-vacuous)', () => {
const fixture = (name, files) => {
const root = join(sandbox, name);
for (const [rel, body] of Object.entries({ ...ABAC_STUB, ...files })) {
for (const [rel, body] of Object.entries({ ...PRELUDE_STUBS, ...files })) {
mkdirSync(dirname(join(root, rel)), { recursive: true });
writeFileSync(join(root, rel), body);
if (rel.endsWith('.mjs')) chmodSync(join(root, rel), 0o755);
Expand Down Expand Up @@ -128,7 +175,7 @@ describe('stale versus out-of-order', () => {
mkdirSync(dirname(join(root, rel)), { recursive: true });
writeFileSync(join(root, rel), body);
};
for (const [rel, body] of Object.entries(ABAC_STUB)) w(rel, body);
for (const [rel, body] of Object.entries(PRELUDE_STUBS)) w(rel, body);
w('reference/core/control-center/gaps/gap-tracking.md', `count: ${boardCount}\n`);
w('reference/core/control-center/maturity-reports/maturity-assessment.md', '# assessment\n');
w('reference/core/control-center/maturity-reports/maturity-reconciliation.json', `{"count":${reconCount}}\n`);
Expand Down Expand Up @@ -177,7 +224,8 @@ fs.writeFileSync(dir + 'executive-summary.es.md', es);
const root = miniRepo('stale-upstream', { boardCount: 9, reconCount: 7, summaryCount: 7 });
const { status, out } = run(root);
assert.equal(status, 1, out);
assert.match(out, /maturity reconciliation is STALE \(link 2 of 3\)/);
const { position, total } = linkPosition('09-reconcile-maturity.mjs');
assert.match(out, new RegExp(`maturity reconciliation is STALE \\(link ${position} of ${total}\\)`));
assert.match(out, /Stopping at the FIRST stale link on purpose/);
assert.doesNotMatch(out, /executive governance summary is STALE/);
});
Expand Down Expand Up @@ -239,7 +287,9 @@ fs.writeFileSync(dir + 'executive-summary.es.md', es);
assert.equal(status, 1, out);
assert.match(out, /differ after replaying the chain IN ORDER/);
assert.match(out, /Fix by regenerating in the declared order/);
// The reconciler is link 2 now that the ABAC rego leads the chain.
assert.match(out, /2\. node \.harness\/scripts\/ci\/09-reconcile-maturity\.mjs/);
// The remediation must list the chain in order, with the reconciler at its
// real position — derived, so adding a link ahead of it does not break this.
const { position } = linkPosition('09-reconcile-maturity.mjs');
assert.match(out, new RegExp(`${position}\\. node \\.harness/scripts/ci/09-reconcile-maturity\\.mjs`));
});
});
Loading
Loading