diff --git a/scripts/check-ratchet-remedy-authority.mjs b/scripts/check-ratchet-remedy-authority.mjs index 96297efa3d..7864a2d7e8 100644 --- a/scripts/check-ratchet-remedy-authority.mjs +++ b/scripts/check-ratchet-remedy-authority.mjs @@ -701,6 +701,40 @@ const CONTROL = { why: 'PR for #11341. Refuses by binding a negation to the verb, over a shrink-only baseline holding only the child-environment leaks that predate the gate; their repair is a card per neighbourhood, so no authority is missing.', }, + // The sixth refusal instance, and the first to arrive because a gate's own + // SHRINK TESTIMONY MOVED rather than because its remedy did (#13489). Its + // remedy sentence is unchanged from before that card and was reachable the + // whole time; what changed is proximity. `anchorFor` reads the ±400 characters + // around every mention of a DECLARED target, and this gate's `⛔ SHRINK-ONLY` + // banner sits at the top of a docblock long enough to push the const's own + // name out of that window. A comment added elsewhere in the file named + // SELF_TEST_RUN_OTHERWISE and its shrink-only nature one line apart, stage 2 + // reached the target for the first time, and a remedy that had been offering + // registry growth for as long as it existed became visible. + // + // ⛔ The lesson is about this detector, not about that comment: a gate whose + // testimony sits just outside the window is INDISTINGUISHABLE here from one + // with no ratchet at all, and the difference is a docblock's length. Recorded + // rather than engineered around — the window is a deliberate precision choice + // (widening it was measured and rejected), so the honest response is to know + // that `excluded` can mean "not reached yet". + // + // Refusal — not marking — because the ledger's own docblock already says it: + // "Nothing joins this list to silence a finding: a gate that really does ship + // an unrun self-test gets wired instead." The author-facing message had been + // contradicting that sentence; it now agrees with it. The legitimate row still + // exists and is named, as a maintainer's judgement of evidence rather than as + // an escape from wiring. Both refusal limbs fire on the rewritten sentence: + // the negation bound to the verb, and the act denied as its own subject. + // + // ⚠️ The two shapes are quoted HERE and not in the `why` below. Putting them + // in the string made this file read as an instance of its own convention and + // reddened self-test case (1) — measured, on the first attempt at this entry. + 'check-self-test-wired.mjs': { + expect: 'refused', + why: 'PR for #13489. Refuses by both limbs at once over a shrink-only ledger — see the sentence quoted in the comment above — because wiring the self-test is the whole remedy and it is the landing author\'s. Reached for the first time when a comment elsewhere in that file put the ledger\'s name and its shrink-only nature one line apart, inside anchorFor\'s window; the remedy sentence itself is older than this card.', + }, + // The tenth, and the fourth to join as a NEW gate rather than by retrofit // (#13582). Marking rather than refusal, on the same reading as the seventh // through ninth: there IS a legitimate act here and it has an owner. The gate diff --git a/scripts/check-self-test-wired.mjs b/scripts/check-self-test-wired.mjs index bc3f6abc09..f02d4261b8 100644 --- a/scripts/check-self-test-wired.mjs +++ b/scripts/check-self-test-wired.mjs @@ -107,6 +107,12 @@ import { maskComments } from './js-comment-mask.mjs'; const ROOT = join(fileURLToPath(new URL('.', import.meta.url)), '..'); const WORKFLOW_DIR = '.github/workflows'; +// The token every gate in this farm writes when it names a path belonging to a +// maintainer rather than to the landing author (#8435). Declared per gate by +// convention, and read out of AUTHOR-FACING strings — a comment mentioning it +// tells an author nothing, so it has to live in the message itself. +const RATCHET_AUTHORITY_MARKER = '⛔ MAINTAINER-ONLY'; + /** Extensions whose files can be a `scripts/` entry point. */ const SCRIPT_EXT = /\.(mjs|mts|js|sh)$/; @@ -263,10 +269,13 @@ export function auditPopulation({ carriers, named, selfTested, ledger }) { ' `--self-test`, but no workflow ever executes it with that flag. An unrun\n' + ' self-test is a phantom check, and for a gate whose defect class is its\n' + ' matching rule it is also the ONLY instrument the rule has (#11150).\n' + - ` Wire \`node ${script} --self-test\` into the step that runs it. If the\n` + - ' self-test is already run some other way — the script drives another tool\'s,\n' + - ' or its ordinary run executes its own cases — add a SELF_TEST_RUN_OTHERWISE\n' + - ' row in scripts/check-self-test-wired.mjs naming the evidence for that.', + ` Wire \`node ${script} --self-test\` into the step that runs it. That is the\n` + + ' whole remedy, and it is the landing author\'s.\n' + + ' ⛔ Do not add a SELF_TEST_RUN_OTHERWISE row to clear this. That ledger is\n' + + ' shrink-only; adding an entry is not the fix, it is this finding written down\n' + + ' somewhere quieter. A row records that a self-test genuinely IS run another\n' + + ' way — a wrapper drives it, or the ordinary run executes its own cases — and\n' + + ` judging that evidence is ${RATCHET_AUTHORITY_MARKER}, never a way out of wiring.`, }); } return findings; @@ -410,14 +419,89 @@ function main() { // --self-test // --------------------------------------------------------------------------- +// --------------------------------------------------------------------------- +// The self-test's own battery registry, floor and verdict (#13489) +// --------------------------------------------------------------------------- +// +// `failures.length === 0` used to be this self-test's ONLY success condition, +// so "every case held" and "the cases never ran" printed the same line. And +// the dispatch below was `if (--self-test) selfTest()`, which discards the +// call's completion: an early `return` anywhere above the verdict printed +// NOTHING and still exited 0. Measured on 597020aa5 by injecting `return;` as +// the first statement of `selfTest()` -- exit 0, zero bytes of output, in the +// gate whose whole subject is that a shipped self-test actually runs. +// +// Both holes are closed the way PR #13487 validated on check-doc-authoring: +// what is pinned is the registered NAMES, not a number. Every section opens +// with `battery('')`, every `ok()` is attributed to the battery most +// recently opened, and the floor requires the OPENED set to equal the DECLARED +// set with each battery at or above its own count. A set difference names +// WHICH battery stopped running; a count says only that something did. +// +// ⛔ A merely non-zero count is not the repair, and neither is a pinned TOTAL: +// a battery dropping from 9 cases to 3 keeps a total "right" for the wrong +// reason the moment a sibling grows. +// +// The counts are a FLOOR, not an equality: adding cases is ordinary work and +// must not red. A battery BELOW its floor means cases stopped running, and the +// remedy is to find what stopped registering -- never to lower the number. +// +// Measured on 597020aa5 by instrumenting `ok` and printing the per-battery +// tally: 7 / 4 / 4 / 4 / 9 / 15, 43 in total. +// +// Five of the six are pinned at what they measure. `live ledger` is not, and +// deliberately: its case count is `1 + one per SELF_TEST_RUN_OTHERWISE row's +// worth`, and that list is ⛔ SHRINK-ONLY by design -- deleting a row is the +// outcome the list exists to reach. A floor at today's 15 would turn every +// legitimate shrink red and train the next author to edit the floor, which is +// the one habit these floors exist to prevent. So what is pinned there is the +// invariant that does NOT move with the list: the structural case ran AND at +// least one row was actually audited (1 + 3). A loop that stops iterating, or +// a ledger that empties without the structural case noticing, still reds. +const SELF_TEST_BATTERIES = Object.freeze({ + 'comment mask': 7, + 'right boundary': 4, + 'alias resolution': 4, + 'population verdict': 4, + 'ledger hygiene': 9, + 'live ledger': 4, +}); + +// DELETING an entry silences that battery's floor exactly as effectively as +// zeroing it, so the registry's own size is pinned too. Adding a battery raises +// this number; removing one is the same ⛔ deliberate edit as lowering a count. +const SELF_TEST_BATTERY_FLOOR = 6; + +// The key an assertion is filed under when no battery is open. It is not a +// declared battery, so it reds by the same set difference rather than silently +// inflating whichever battery happened to run last. +const UNATTRIBUTED_BATTERY = '(no battery open)'; + +// Returned by `selfTest()` only after the floor has been evaluated and the +// verdict printed. The dispatch refuses anything else: a `return` that leaves +// the function early prints nothing and exits 0, which is the same +// nothing-ran-nothing-complained pass one level up. +const SELF_TEST_VERDICT = 'check-self-test-wired self-test reached its verdict'; + function selfTest() { const failures = []; + const seen = new Map(); + let openBattery = null; + // Declare the battery the following assertions belong to. The name must be a + // key of SELF_TEST_BATTERIES -- an unknown one reds by set difference, + // naming itself, rather than being counted somewhere it is not floored. + const battery = (name) => { + openBattery = name; + }; const ok = (cond, label) => { + const b = openBattery ?? UNATTRIBUTED_BATTERY; + seen.set(b, (seen.get(b) ?? 0) + 1); if (!cond) failures.push(label); }; const wf = (text, name = 'lint.yml') => [{ name, text }]; // ── Prose never decides anything, in either direction ──────────────────── + battery('comment mask'); ok( !carriesSelfTest('scripts/x.mjs', '// run it with --self-test sometimes\nconst a = 1;\n'), 'a `--self-test` that exists only in a JS comment was read as an implementation', @@ -447,6 +531,7 @@ function selfTest() { } // ── Right boundary: the defect class this gate is itself in the family for ─ + battery('right boundary'); { const got = collectInvocations(wf(' - run: node scripts/g.mjs --self-test-extra\n'), {}); ok(got.named.has('scripts/g.mjs'), 'the script was not seen at all — the boundary case would test nothing'); @@ -468,6 +553,7 @@ function selfTest() { } // ── Aliases: reached only when a workflow actually names them ──────────── + battery('alias resolution'); { const pkg = { 'check:thing': 'node scripts/thing.mjs --self-test && node scripts/thing.mjs', @@ -491,6 +577,7 @@ function selfTest() { } // ── The population verdict, both directions ────────────────────────────── + battery('population verdict'); { const carriers = new Set(['scripts/g.mjs']); const run = (text, ledger = []) => { @@ -511,6 +598,7 @@ function selfTest() { } // ── Ledger hygiene: every row must still be true, and still be needed ──── + battery('ledger hygiene'); { const carriers = new Set(['scripts/w.mjs', 'scripts/t.mjs']); const sources = { @@ -561,6 +649,7 @@ function selfTest() { } // ── The live ledger, checked against the real tree ─────────────────────── + battery('live ledger'); { const sourceOf = (relPath) => { try { @@ -582,18 +671,79 @@ function selfTest() { } } + // ── The floor: every declared battery RAN, and ran its cases ───────────── + // + // Evaluated here, after every battery has had its chance and BEFORE the + // verdict -- so the line below can only be printed by a run in which the set + // of batteries that registered assertions equals the set declared. + const declaredBatteries = Object.keys(SELF_TEST_BATTERIES); + const totalCases = [...seen.values()].reduce((a, b) => a + b, 0); + let floorBreached = false; + if (declaredBatteries.length < SELF_TEST_BATTERY_FLOOR) { + floorBreached = true; + failures.push( + `SELF_TEST_BATTERIES declares ${declaredBatteries.length} batteries, below the pinned ` + + `${SELF_TEST_BATTERY_FLOOR} — a battery deleted from the registry takes its own floor with it.`, + ); + } + for (const [name, count] of seen) { + if (declaredBatteries.includes(name)) continue; + floorBreached = true; + failures.push( + `self-test battery "${name}" registered ${count} case(s) but is not declared in ` + + 'SELF_TEST_BATTERIES — an assertion attributed to no declared battery is one nothing floors.', + ); + } + for (const name of declaredBatteries) { + const count = seen.get(name) ?? 0; + if (count >= SELF_TEST_BATTERIES[name]) continue; + floorBreached = true; + failures.push( + count === 0 + ? `self-test battery "${name}" DID NOT RUN — 0 cases registered, ${SELF_TEST_BATTERIES[name]} pinned. ` + + 'The verdict below would have claimed they hold.' + : `self-test battery "${name}" registered ${count} case(s), below its pinned floor of ` + + `${SELF_TEST_BATTERIES[name]} — cases that used to run no longer do.`, + ); + } + if (failures.length > 0) { console.error('check-self-test-wired --self-test FAILED:'); for (const f of failures) console.error(` - ${f}`); + if (floorBreached) { + console.error( + '\nA battery at or below its floor means cases STOPPED RUNNING — the battery is the bug,\n' + + 'not the number. Find what stopped registering (an early return, a deleted block, a guard\n' + + 'that now skips) and restore it. Raising a floor after ADDING cases is ordinary work;\n' + + `LOWERING one is ${RATCHET_AUTHORITY_MARKER}, not a co-equal option — "the count\n` + + 'legitimately moved" and "something stopped running" need different edits, and only a\n' + + 'measurement tells them apart.\n', + ); + } process.exit(1); } console.log( `check-self-test-wired --self-test: ${SELF_TEST_RUN_OTHERWISE.length} live ledger row(s) verified, plus the ` + - 'comment mask, the right boundary, alias resolution and both audit directions.', + 'comment mask, the right boundary, alias resolution and both audit directions' + + ` — ${declaredBatteries.length} declared batteries, ${totalCases} cases registered, every battery at or` + + ' above its pinned floor.', ); + return SELF_TEST_VERDICT; } if (isEntrypoint(import.meta.url)) { - if (process.argv.includes('--self-test')) selfTest(); - else main(); + if (process.argv.includes('--self-test')) { + // ⛔ Never `selfTest();` bare, and never `return selfTest()`. A `return` + // anywhere above that verdict prints nothing, evaluates no floor and exits + // 0 — the same nothing-ran-nothing-complained pass the battery floor + // refuses, one level up (#13489). + if (selfTest() !== SELF_TEST_VERDICT) { + console.error( + '\n✗ check-self-test-wired self-test: selfTest() returned without reaching its verdict, so no\n' + + 'battery floor was evaluated and no success line was printed. Exiting 0 here would report a\n' + + 'self-test that never finished as a self-test that passed.\n', + ); + process.exit(1); + } + } else main(); } diff --git a/scripts/check-self-test-workflow-commands.mjs b/scripts/check-self-test-workflow-commands.mjs index e9047b10d9..f814a03729 100644 --- a/scripts/check-self-test-workflow-commands.mjs +++ b/scripts/check-self-test-workflow-commands.mjs @@ -309,9 +309,69 @@ function main() { // only a label, never the offending line. // --------------------------------------------------------------------------- +// --------------------------------------------------------------------------- +// The self-test's own battery registry, floor and verdict (#13489) +// --------------------------------------------------------------------------- +// +// `failures.length === 0` used to be this self-test's ONLY success condition, +// so "every case held" and "the cases never ran" printed the same line. And +// the dispatch below was `if (--self-test) selfTest()`, which discards the +// call's completion: an early `return` anywhere above the verdict printed +// NOTHING and still exited 0. Measured on 597020aa5 by injecting `return;` as +// the first statement of `selfTest()` -- exit 0, zero bytes of output. +// +// Both holes are closed the way PR #13487 validated on check-doc-authoring: +// what is pinned is the registered NAMES, not a number. Every section opens +// with `battery('')`, every `ok()` is attributed to the battery most +// recently opened, and the floor requires the OPENED set to equal the DECLARED +// set with each battery at or above its own count. A set difference names +// WHICH battery stopped running; a count says only that something did -- and a +// pinned TOTAL rots the moment a sibling battery grows. +// +// The counts are a FLOOR, not an equality: adding cases is ordinary work and +// must not red. A battery BELOW its floor means cases stopped running, and the +// remedy is to find what stopped registering -- never to lower the number. +// +// Measured on 597020aa5 by instrumenting `ok` and printing the per-battery +// tally. +const SELF_TEST_BATTERIES = Object.freeze({ + 'the measured defect': 2, + 'legacy form, anywhere in a line': 4, + 'current form, line start only': 3, + 'innocent output': 4, + 'prefilter reads CODE, never prose': 5, + 'end to end on the real defect site': 5, +}); + +// DELETING an entry silences that battery's floor exactly as effectively as +// zeroing it, so the registry's own size is pinned too. Adding a battery raises +// this number; removing one is the same ⛔ deliberate edit as lowering a count. +const SELF_TEST_BATTERY_FLOOR = 6; + +// The key an assertion is filed under when no battery is open. It is not a +// declared battery, so it reds by the same set difference rather than silently +// inflating whichever battery happened to run last. +const UNATTRIBUTED_BATTERY = '(no battery open)'; + +// Returned by `selfTest()` only after the floor has been evaluated and the +// verdict printed. The dispatch refuses anything else: a `return` that leaves +// the function early prints nothing and exits 0, which is the same +// nothing-ran-nothing-complained pass one level up. +const SELF_TEST_VERDICT = 'check-self-test-workflow-commands self-test reached its verdict'; + function selfTest() { const failures = []; + const seen = new Map(); + let openBattery = null; + // Declare the battery the following assertions belong to. The name must be a + // key of SELF_TEST_BATTERIES -- an unknown one reds by set difference, + // naming itself, rather than being counted somewhere it is not floored. + const battery = (name) => { + openBattery = name; + }; const ok = (cond, label) => { + const b = openBattery ?? UNATTRIBUTED_BATTERY; + seen.set(b, (seen.get(b) ?? 0) + 1); if (!cond) failures.push(label); }; const HASHES = '#'.repeat(2); @@ -320,6 +380,7 @@ function selfTest() { const v2 = (name) => `${COLONS}${name}${COLONS}`; // ── The measured defect itself, reproduced from the real annotation ────── + battery('the measured defect'); { const real = ` tail with no \`${v1('error')}\` in it is labelled a window rather than an anchor. And a \`fix\``; const got = scanOutput(real); @@ -328,12 +389,14 @@ function selfTest() { } // ── Measurement 1: the legacy form is parsed ANYWHERE in a line ────────── + battery('legacy form, anywhere in a line'); ok(scanOutput(`${v1('error')}boom`).length === 1, 'a legacy command at column 0 was not flagged'); ok(scanOutput(`prose about ${v1('error')} here`).length === 1, 'a legacy command MID-LINE was not flagged — the measured defect walks straight through'); ok(scanOutput(` ${v1('warning')} x`).length === 1, 'an indented legacy command was not flagged'); ok(scanOutput(`x ${v1('group')} y`).length === 1, 'a legacy `group` was not flagged'); // ── Measurement 2: the current form is parsed only at LINE START ───────── + battery('current form, line start only'); ok(scanOutput(`${v2('error')}boom`).length === 1, 'a current-form command at line start was not flagged'); ok(scanOutput(`${COLONS}error file=a.ts,line=1${COLONS}boom`).length === 1, 'a current-form command WITH PROPERTIES was not flagged'); ok( @@ -342,12 +405,14 @@ function selfTest() { ); // ── Innocent output must stay innocent, or the gate gets weakened ──────── + battery('innocent output'); ok(scanOutput('✓ check-foo: 12 file(s) scanned, nothing to report').length === 0, 'an ordinary success line was flagged'); ok(scanOutput(' see docs/x.md ## Heading and packages/spec [ok]').length === 0, 'a markdown heading plus a bracket was flagged'); ok(scanOutput(`Time${COLONS}HiRes${COLONS}time()`).length === 0, 'a Perl namespace was flagged as a workflow command'); ok(scanOutput('').length === 0, 'empty output produced a finding'); // ── The prefilter selects on CODE, never on prose ──────────────────────── + battery('prefilter reads CODE, never prose'); ok( !isCandidate('scripts/x.mjs', `// the runner spells it ${v1('error')}\nconst a = 1;\n`), 'a token that exists only in a JS comment selected the script — comments are never printed', @@ -370,6 +435,7 @@ function selfTest() { ); // ── End to end: the real tree's real defect site, run for real ─────────── + battery('end to end on the real defect site'); { const target = 'scripts/pm/ci-failure.mjs'; const source = existsSync(join(ROOT, target)) ? readFileSync(join(ROOT, target), 'utf8') : null; @@ -381,19 +447,79 @@ function selfTest() { ok(scanOutput(r.output).length === 0, `${target} --self-test still prints a line the runner would parse — this is the #11886 defect, live`); } + // ── The floor: every declared battery RAN, and ran its cases ───────────── + // + // Evaluated here, after every battery has had its chance and BEFORE the + // verdict -- so the line below can only be printed by a run in which the set + // of batteries that registered assertions equals the set declared. + const declaredBatteries = Object.keys(SELF_TEST_BATTERIES); + const totalCases = [...seen.values()].reduce((a, b) => a + b, 0); + let floorBreached = false; + if (declaredBatteries.length < SELF_TEST_BATTERY_FLOOR) { + floorBreached = true; + failures.push( + `SELF_TEST_BATTERIES declares ${declaredBatteries.length} batteries, below the pinned ` + + `${SELF_TEST_BATTERY_FLOOR} — a battery deleted from the registry takes its own floor with it.`, + ); + } + for (const [name, count] of seen) { + if (declaredBatteries.includes(name)) continue; + floorBreached = true; + failures.push( + `self-test battery "${name}" registered ${count} case(s) but is not declared in ` + + 'SELF_TEST_BATTERIES — an assertion attributed to no declared battery is one nothing floors.', + ); + } + for (const name of declaredBatteries) { + const count = seen.get(name) ?? 0; + if (count >= SELF_TEST_BATTERIES[name]) continue; + floorBreached = true; + failures.push( + count === 0 + ? `self-test battery "${name}" DID NOT RUN — 0 cases registered, ${SELF_TEST_BATTERIES[name]} pinned. ` + + 'The verdict below would have claimed they hold.' + : `self-test battery "${name}" registered ${count} case(s), below its pinned floor of ` + + `${SELF_TEST_BATTERIES[name]} — cases that used to run no longer do.`, + ); + } + if (failures.length > 0) { console.error('check-self-test-workflow-commands --self-test FAILED:'); for (const f of failures) console.error(` - ${f}`); + if (floorBreached) { + console.error( + '\nA battery at or below its floor means cases STOPPED RUNNING — the battery is the bug,\n' + + 'not the number. Find what stopped registering (an early return, a deleted block, a guard\n' + + 'that now skips) and restore it. Raising a floor after ADDING cases is ordinary work;\n' + + 'LOWERING one is not a co-equal option — "the count legitimately moved" and "something\n' + + 'stopped running" need different edits, and only a measurement tells them apart.\n', + ); + } process.exit(1); } console.log( 'check-self-test-workflow-commands --self-test: both measured parse rules pinned (legacy form ' + 'anywhere in a line, current form only at line start), the innocent-output and Perl-namespace ' + - 'cases, the comment mask in both directions, and one end-to-end run of the real defect site.', + 'cases, the comment mask in both directions, and one end-to-end run of the real defect site' + + ` — ${declaredBatteries.length} declared batteries, ${totalCases} cases registered, every battery` + + ' at or above its pinned floor.', ); + return SELF_TEST_VERDICT; } if (isEntrypoint(import.meta.url)) { - if (process.argv.includes('--self-test')) selfTest(); - else main(); + if (process.argv.includes('--self-test')) { + // ⛔ Never `selfTest();` bare, and never `return selfTest()`. A `return` + // anywhere above that verdict prints nothing, evaluates no floor and exits + // 0 — the same nothing-ran-nothing-complained pass the battery floor + // refuses, one level up (#13489). + if (selfTest() !== SELF_TEST_VERDICT) { + console.error( + '\n✗ check-self-test-workflow-commands self-test: selfTest() returned without reaching its\n' + + 'verdict, so no battery floor was evaluated and no success line was printed. Exiting 0 here\n' + + 'would report a self-test that never finished as a self-test that passed.\n', + ); + process.exit(1); + } + } else main(); } diff --git a/scripts/measure-self-test-floor.mjs b/scripts/measure-self-test-floor.mjs new file mode 100644 index 0000000000..bf55400cc1 --- /dev/null +++ b/scripts/measure-self-test-floor.mjs @@ -0,0 +1,410 @@ +#!/usr/bin/env node +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * measure-self-test-floor -- can each `scripts/**` self-test prove it ran? (#13489) + * + * node scripts/measure-self-test-floor.mjs # static census (fast) + * node scripts/measure-self-test-floor.mjs --probe # + the dynamic probe (minutes) + * node scripts/measure-self-test-floor.mjs --json # machine-readable + * + * ## The two holes, which are ORTHOGONAL + * + * A gate can be clean on one and defeated by the other, so they are counted + * separately and never summed into one "problem gates" total. + * + * 1. NO ASSERTION FLOOR. Success decided by `failures.length === 0` alone, + * so "every case held" and "the cases never ran" print the same line. + * 2. NO VERDICT HANDSHAKE. The dispatch discards the self-test's completion + * (`return selfTest()`, `selfTest();`, `process.exit(selfTest())`), so a + * `return` anywhere above the verdict prints NOTHING and still exits 0. + * A gate with a perfect floor is still defeated this way: the floor never + * runs either. + * + * ⚠️ A verdict line that already prints a case count is EVIDENCE, NOT PROOF. A + * battery dropping from 40 cases to 3 still prints a non-zero count and passes, + * and pinning a TOTAL rots the moment a sibling battery grows. Two gates in + * this tree derive and print a `SELF_TEST_CASE_COUNT` that nothing ever + * compares; both classify NONE here, correctly. + * + * ## Boundary: this is NOT the empty-scan class + * + * "A sweep that read zero must refuse" is a different property. `check-adr-links` + * and `check-doc-anchors` both carry that refusal AND are defeated by hole 2 -- + * measured, in this tree. Neither class covers for the other. + * + * ## Why hole 2 is MEASURED and hole 1 is READ + * + * The grep the triage ruling supplies (`failures.length === 0`, `return + * selfTest()`) is an ENTRY POINT, not a criterion: a gate reaches the same + * effect through `process.exit(selfTest())` (an early bare `return` yields + * `undefined`, and `process.exit(undefined)` is exit 0), through + * `selfTest(); main();`, or through a top-level block with no callee at all. + * So hole 2 is decided by BEHAVIOUR: inject `return;` as the first statement of + * the function the dispatch calls, run it, read the exit code. Exit 0 is the + * defect. Hole 1 has no equally generic mutation -- a battery is not a + * mechanically identifiable unit across 158 differently shaped self-tests -- so + * it is decided by a published static criterion instead, stated below. + * + * ## The controls, which run on EVERY invocation + * + * This tool's whole subject is "a green that asserted nothing". A survey that + * silently misses a class of files and reports zero commits exactly that + * defect. So both instruments are driven against KNOWN-HOLED and KNOWN-SOUND + * fixtures before any number is printed, and a control failure refuses -- it + * does not degrade to a smaller number. They are placed here, unconditionally, + * rather than behind a `--self-test` flag, precisely so they cannot become + * unrun; that is the `inline` route `check-self-test-wired.mjs` records. + * + * The controls have already earned their place once: an earlier revision of + * `classifyFloor` keyed on the NAME `SELF_TEST_BATTERIES` rather than on a + * comparison that produces a failure, and called a fixture floored after the + * roster had been removed. The control caught it; nothing else would have. + */ + +import { readFileSync, writeFileSync, rmSync, readdirSync, existsSync, mkdtempSync } from 'node:fs'; +import { spawnSync } from 'node:child_process'; +import { join, dirname, basename, sep } from 'node:path'; +import { tmpdir } from 'node:os'; +import { fileURLToPath } from 'node:url'; + +import { isEntrypoint } from './invoked-as.mjs'; +import { maskComments } from './js-comment-mask.mjs'; + +const ROOT = join(fileURLToPath(new URL('.', import.meta.url)), '..'); + +/** A `--self-test` DISPATCH: argv membership tested, not a literal passed to a child. */ +const DISPATCH = /(?:includes|has)\(\s*['"`]--self-test['"`]\s*\)/; + +/** Marker injected by the probe. Its presence on disk is the mutation's proof. */ +const PROBE_MARKER = 'OS_SELF_TEST_FLOOR_PROBE'; + +// --------------------------------------------------------------------------- +// Instrument 1 -- the static assertion-floor criterion +// --------------------------------------------------------------------------- + +/** + * A FLOOR must PRODUCE A FAILURE, not merely be named. Keying on a name is the + * mistake this card is about one level up, and the control below proves it. + */ +const PRODUCES_FAILURE = /failures\.push|process\.exit\(1\)|throw new Error|exitCode = 1|ok\(false/; +const ROSTER_COMPARISON = + /(?:declaredBatteries|SELF_TEST_BATTERIES|BATTERY_FLOOR)[^;]{0,400}?(?:\.length|\.size|includes\(|has\(|!==|===|<)/s; +const COUNT_COMPARISON = [ + new RegExp( + String.raw`\b(?:checked|cases|caseCount|ran|seen|asserted|assertions|count|total|CASES|CHECKED)\w*` + + String.raw`(?:\.(?:length|size))?\s*(?:<|!==|!=|<=)\s*(?:\d+|[A-Z][A-Z0-9_]{3,})`, + ), + new RegExp( + String.raw`(?:\d+|[A-Z][A-Z0-9_]{3,})\s*(?:>|!==|!=|>=)\s*` + + String.raw`\b(?:checked|cases|caseCount|ran|seen|asserted|assertions|count|total)\w*(?:\.(?:length|size))?`, + ), +]; + +/** + * ROSTER -- declared battery NAMES compared as a set (the #13487 shape). + * COUNT -- a registered count compared against a declared constant. + * NONE -- success decided by "no failure was recorded", and nothing else. + * + * ⚠️ The criterion reads NAMES (`SELF_TEST_BATTERIES`, `declaredBatteries`, a + * counter called `checked`/`cases`/...). A floor spelled with names it does not + * know reads as NONE, so its error runs in ONE direction: it can call a floored + * self-test unfloored, never the reverse. A NONE is therefore a candidate to + * read, and the population it reports is an UPPER bound on the hole. On + * 597020aa5 the tree was also hand-swept for zero-case refusals independently + * of these names; every hit was a production-scan refusal (the adjacent + * empty-scan class), not a self-test floor. + * + * Deliberately high-recall: COUNT hits are candidates to READ, not verdicts. + * Both COUNT hits in this tree on 597020aa5 were hand-checked and are false + * positives (`count < 100` in a production probe; a `total < 0` sign test). + */ +export function classifyFloor(code) { + if (ROSTER_COMPARISON.test(code) && PRODUCES_FAILURE.test(code)) return 'ROSTER'; + if (COUNT_COMPARISON.some((re) => re.test(code)) && PRODUCES_FAILURE.test(code)) return 'COUNT'; + return 'NONE'; +} + +// --------------------------------------------------------------------------- +// Instrument 2 -- the dynamic verdict-handshake probe +// --------------------------------------------------------------------------- + +/** Every `/self.?test/i`-named function DEFINED in this source. */ +export function selfTestDefs(src) { + const names = new Set(); + for (const m of src.matchAll(/(?:async\s+)?function\s+([A-Za-z_$][\w$]*)\s*\(/g)) { + if (/self.?test/i.test(m[1])) names.add(m[1]); + } + for (const m of src.matchAll(/const\s+([A-Za-z_$][\w$]*)\s*=\s*(?:async\s*)?(?:\([^)]*\)|[A-Za-z_$][\w$]*)\s*=>/g)) { + if (/self.?test/i.test(m[1])) names.add(m[1]); + } + return [...names]; +} + +/** Insert `return;` as the first statement of `name`. Returns null when absent. */ +export function injectEarlyReturn(src, name) { + const pats = [ + new RegExp(`(?:async\\s+)?function\\s+${name}\\s*\\([^)]*\\)\\s*(?::\\s*[A-Za-z_$][\\w$<>\\[\\]|. ]*\\s*)?\\{`), + new RegExp(`const\\s+${name}\\s*=\\s*(?:async\\s*)?\\([^)]*\\)\\s*(?::[^=]*)?=>\\s*\\{`), + ]; + for (const re of pats) { + const m = src.match(re); + if (!m) continue; + const at = m.index + m[0].length; + return `${src.slice(0, at)}\n return; /*${PROBE_MARKER}*/\n${src.slice(at)}`; + } + return null; +} + +/** + * Run one gate's `--self-test` with an early `return` at the top of `entry`. + * + * The copy is written BESIDE the original so relative imports and repo-root + * resolution still answer the same, and the marker is re-read FROM DISK before + * the run: an editor step that matched nothing exits 0 just as happily as one + * that landed, and an unmutated file would report "held" for no reason at all. + */ +export function probeEarlyReturn(absFile, entry, { timeout = 120000 } = {}) { + const src = readFileSync(absFile, 'utf8'); + const mutated = injectEarlyReturn(src, entry); + if (mutated === null) return { verdict: 'NOT MEASURED', why: `no injectable definition of ${entry}` }; + if (src.includes(PROBE_MARKER)) return { verdict: 'NOT MEASURED', why: 'marker already present in source' }; + + const probePath = join(dirname(absFile), `.self-test-floor-probe-${basename(absFile)}`); + const isTs = /\.(mts|ts)$/.test(absFile); + const cmd = isTs ? join(ROOT, 'node_modules/.bin/tsx') : process.execPath; + try { + writeFileSync(probePath, mutated); + const onDisk = (readFileSync(probePath, 'utf8').match(new RegExp(PROBE_MARKER, 'g')) ?? []).length; + if (onDisk !== 1) return { verdict: 'NOT MEASURED', why: `mutation not on disk (marker x${onDisk})` }; + + const base = spawnSync(cmd, [absFile, '--self-test'], { cwd: ROOT, timeout, encoding: 'utf8' }); + const mut = spawnSync(cmd, [probePath, '--self-test'], { cwd: ROOT, timeout, encoding: 'utf8' }); + const baseOut = (base.stdout ?? '') + (base.stderr ?? ''); + const mutOut = (mut.stdout ?? '') + (mut.stderr ?? ''); + if (mut.signal || base.signal) return { verdict: 'NOT MEASURED', why: `killed by ${mut.signal ?? base.signal}` }; + // A mutation that changed nothing observable did not reach the executed + // path, whatever its exit code says. + if (baseOut === mutOut && base.status === mut.status) { + return { verdict: 'NOT MEASURED', why: 'mutation had no observable effect' }; + } + return { + verdict: mut.status === 0 ? 'DEFEATED' : 'HELD', + entry, + baselineExit: base.status, + mutatedExit: mut.status, + mutatedBytes: mutOut.length, + mutatedHead: mutOut.split('\n').find((l) => l.trim()) ?? '', + }; + } finally { + rmSync(probePath, { force: true }); + } +} + +// --------------------------------------------------------------------------- +// The controls -- run on EVERY invocation, before any number is printed +// --------------------------------------------------------------------------- + +const HOLED_GATE = [ + '#!/usr/bin/env node', + 'function selfTest() {', + ' const failures = [];', + " if (1 !== 1) failures.push('x');", + " if (failures.length) { console.error('nope'); process.exit(1); }", + " console.log('fixture self-test: 1 case passes');", + '}', + "if (process.argv.includes('--self-test')) selfTest();", + '', +].join('\n'); + +const SOUND_GATE = [ + '#!/usr/bin/env node', + "const VERDICT = 'reached';", + "const SELF_TEST_BATTERIES = Object.freeze({ only: 1 });", + 'const SELF_TEST_BATTERY_FLOOR = 1;', + 'function selfTest() {', + ' const failures = [];', + ' const seen = new Map();', + " let open = null;", + ' const battery = (n) => { open = n; };', + " const ok = (c, l) => { seen.set(open, (seen.get(open) ?? 0) + 1); if (!c) failures.push(l); };", + " battery('only');", + " ok(1 === 1, 'x');", + ' const declaredBatteries = Object.keys(SELF_TEST_BATTERIES);', + " if (declaredBatteries.length < SELF_TEST_BATTERY_FLOOR) failures.push('registry shrank');", + " for (const n of declaredBatteries) if ((seen.get(n) ?? 0) < SELF_TEST_BATTERIES[n]) failures.push('battery ' + n + ' did not run');", + ' if (failures.length) { console.error(failures.join(String.fromCharCode(10))); process.exit(1); }', + " console.log('fixture self-test: floor held');", + ' return VERDICT;', + '}', + "if (process.argv.includes('--self-test')) {", + ' if (selfTest() !== VERDICT) {', + " console.error('fixture: selfTest returned without reaching its verdict');", + ' process.exit(1);', + ' }', + '}', + '', +].join('\n'); + +/** + * Both instruments, against both directions. Returns the failures; the caller + * refuses on any. Nothing here reads the repo, so a control failure is always + * the instrument and never the tree. + */ +export function runControls() { + const failures = []; + const say = (cond, label) => { if (!cond) failures.push(label); }; + + // Instrument 1, both directions. + say(classifyFloor(maskComments(HOLED_GATE)) === 'NONE', + 'POSITIVE CONTROL FAILED: a self-test deciding success by failures.length alone was not classified NONE'); + say(classifyFloor(maskComments(SOUND_GATE)) === 'ROSTER', + 'NEGATIVE CONTROL FAILED: a roster-floored self-test was not classified ROSTER'); + + // Instrument 2, both directions, against real processes on disk. + const dir = mkdtempSync(join(tmpdir(), 'self-test-floor-control-')); + try { + const holed = join(dir, 'holed-gate.mjs'); + const sound = join(dir, 'sound-gate.mjs'); + writeFileSync(holed, HOLED_GATE); + writeFileSync(sound, SOUND_GATE); + const h = probeEarlyReturn(holed, 'selfTest'); + const s = probeEarlyReturn(sound, 'selfTest'); + say(h.verdict === 'DEFEATED', + `POSITIVE CONTROL FAILED: the probe read a known-holed gate as ${h.verdict} (${h.why ?? ''})`); + say(h.mutatedBytes === 0, + 'POSITIVE CONTROL FAILED: the known-holed gate printed something; the measured shape prints NOTHING'); + say(s.verdict === 'HELD', + `NEGATIVE CONTROL FAILED: the probe read a handshake-protected gate as ${s.verdict} (${s.why ?? ''})`); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + return failures; +} + +// --------------------------------------------------------------------------- +// The census +// --------------------------------------------------------------------------- + +/** + * ⛔ SHRINK-ONLY. The nine files whose entry cannot be resolved mechanically, + * resolved by READING the dispatch site (the ruling's A2.1: the grep is an + * entry point, not the criterion). A `null` entry is NOT MEASURED with the + * stated reason -- never a quiet pass, and never a guess. + * + * A row whose file no longer holds more than one `/self.?test/i` definition is + * dead and should be deleted; the mechanical path then covers it. + */ +export const ENTRY_BY_HAND = Object.freeze({ + 'scripts/check-comment-mask-corpus.mjs': 'selfTest', + 'scripts/check-doc-authoring.mjs': 'selfTest', + 'scripts/check-durability-degradation-log-level.mjs': 'selfTest', + 'scripts/check-self-test-wired.mjs': 'selfTest', + 'scripts/check-self-test-workflow-commands.mjs': 'selfTest', + 'scripts/check-turbo-task-graph.mjs': 'runSelfTest', + // The dispatch calls FOUR self-test functions and combines their statuses; + // there is no single entry an early return leaves, so a one-function probe + // measures a sub-battery and reads a downstream crash as a handshake. + 'scripts/check-platform-checklist.mjs': null, + // The self-test is an inline top-level block calling several helpers. + 'scripts/check-regen-pending.mjs': null, + // Injecting into this file produces a SyntaxError (the anchor lands inside a + // template literal), so no run of it measures anything. + 'scripts/pm/dispatch-gates.mjs': null, +}); + +/** + * This file is not itself a member: the `--self-test` literals below live in + * CONTROL FIXTURE strings, which are data, not a dispatch. It deliberately + * ships no `--self-test` mode -- its controls run inline on every invocation, + * so they cannot become unrun. + */ +const CENSUS_SELF = 'scripts/measure-self-test-floor.mjs'; + +function walk(dir, out = []) { + for (const e of readdirSync(dir, { withFileTypes: true })) { + const p = join(dir, e.name); + if (e.isDirectory()) { + if (e.name === 'node_modules') continue; + walk(p, out); + } else if (/\.(mjs|mts|js|ts)$/.test(e.name)) out.push(p); + } + return out; +} + +/** Every `scripts/**` file that DISPATCHES on `--self-test`, read from CODE. */ +export function population() { + const scripts = join(ROOT, 'scripts'); + if (!existsSync(scripts)) throw new Error('scripts/ does not resolve -- the census would report zero for the wrong reason'); + const rows = []; + for (const abs of walk(scripts).sort()) { + const src = readFileSync(abs, 'utf8'); + const code = maskComments(src); + if (!DISPATCH.test(code)) continue; + const file = abs.slice(ROOT.length + 1).split(sep).join('/'); + if (file === CENSUS_SELF) continue; + rows.push({ file, abs, floor: classifyFloor(code), defs: selfTestDefs(src) }); + } + if (rows.length === 0) throw new Error('the census found no self-test dispatch at all -- refusing rather than reporting zero'); + return rows; +} + +function main() { + const controlFailures = runControls(); + if (controlFailures.length > 0) { + console.error('measure-self-test-floor: ITS OWN CONTROLS FAILED -- no census printed.\n'); + for (const f of controlFailures) console.error(` - ${f}`); + console.error('\nA survey whose instrument cannot see a known hole reports zero for the same reason'); + console.error('this measurement exists. Fix the instrument; a smaller number is not the fallback.\n'); + process.exit(1); + } + + const rows = population(); + const wantProbe = process.argv.includes('--probe'); + if (wantProbe) { + for (const r of rows) { + const named = Object.hasOwn(ENTRY_BY_HAND, r.file) ? ENTRY_BY_HAND[r.file] : undefined; + if (named === null) { r.probe = { verdict: 'NOT MEASURED', why: 'entry read by hand as not probeable -- see ENTRY_BY_HAND' }; continue; } + const entry = named ?? (r.defs.length === 1 ? r.defs[0] : undefined); + if (entry === undefined) { + r.probe = { verdict: 'NOT MEASURED', why: r.defs.length === 0 + ? 'self-test is an inline top-level block; no callee to leave early' + : `ambiguous entry (${r.defs.join(', ')}) and no ENTRY_BY_HAND row -- read the dispatch site` }; + continue; + } + r.probe = probeEarlyReturn(r.abs, entry); + } + } + + if (process.argv.includes('--json')) { + console.log(JSON.stringify(rows.map(({ abs, ...rest }) => rest), null, 2)); + return; + } + + const byFloor = { ROSTER: [], COUNT: [], NONE: [] }; + for (const r of rows) byFloor[r.floor].push(r.file); + console.log(`measure-self-test-floor: ${rows.length} file(s) under scripts/ dispatch on \`--self-test\`.\n`); + console.log('Hole 1 -- no assertion floor (success decided by "no failure was recorded"):'); + console.log(` ${byFloor.NONE.length} of ${rows.length}. Floored: ${byFloor.ROSTER.length} roster, ${byFloor.COUNT.length} count-candidate(s) to read.`); + if (byFloor.ROSTER.length) console.log(` roster: ${byFloor.ROSTER.join(', ')}`); + if (byFloor.COUNT.length) console.log(` count candidates: ${byFloor.COUNT.join(', ')}`); + + if (!wantProbe) { + console.log('\nHole 2 -- no verdict handshake: NOT MEASURED (pass --probe; it runs every self-test twice).'); + return; + } + const defeated = rows.filter((r) => r.probe.verdict === 'DEFEATED'); + const held = rows.filter((r) => r.probe.verdict === 'HELD'); + const unmeasured = rows.filter((r) => r.probe.verdict === 'NOT MEASURED'); + console.log('\nHole 2 -- silently defeated by an early `return` in the self-test (MEASURED):'); + console.log(` ${defeated.length} DEFEATED, ${held.length} HELD, ${unmeasured.length} NOT MEASURED.`); + console.log(` of the defeated, ${defeated.filter((r) => r.probe.mutatedBytes === 0).length} printed NOTHING at all and still exited 0.`); + for (const r of held) console.log(` HELD ${r.file} -- ${r.probe.mutatedHead.slice(0, 96)}`); + for (const r of unmeasured) console.log(` n/m ${r.file} -- ${r.probe.why}`); + console.log('\n⛔ The two numbers are ORTHOGONAL and are never summed: a gate with a perfect'); + console.log(' floor is still defeated by hole 2, because the floor never runs either.'); +} + +if (isEntrypoint(import.meta.url)) { + main(); +}