|
| 1 | +#!/usr/bin/env node |
| 2 | +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. |
| 3 | +// |
| 4 | +// ablation-dist-preflight -- prove an ablation actually reached the BUILT |
| 5 | +// artifact before the ablation run's colour is allowed to mean anything. |
| 6 | +// |
| 7 | +// node scripts/ablation-dist-preflight.mjs <package> <marker> |
| 8 | +// node scripts/ablation-dist-preflight.mjs <package> <marker> --absent |
| 9 | +// node scripts/ablation-dist-preflight.mjs --self-test |
| 10 | +// |
| 11 | +// ## The failure this exists to stop |
| 12 | +// |
| 13 | +// `packages/qa/dogfood` resolves the code under test from each package's built |
| 14 | +// `dist/`, not `src/` -- deliberately, because that is what covers packaging and |
| 15 | +// export-surface defects. Editing `src` therefore has no effect on the suite |
| 16 | +// until that package is rebuilt, and the two directions of forgetting are NOT |
| 17 | +// equally dangerous: |
| 18 | +// |
| 19 | +// forgot to rebuild a FIX -> false RED. Costs a lap, and gets noticed. |
| 20 | +// forgot to rebuild an ABLATION -> false GREEN. Silently certifies a vacuous |
| 21 | +// test as "verified discriminating". |
| 22 | +// |
| 23 | +// The second one is why this script exists. An ablation's whole purpose is to |
| 24 | +// show the test goes red when the defect is present; run against the |
| 25 | +// pre-mutation build it stays green, that green is written down as "ablation |
| 26 | +// done, direction as predicted", and an assertion that may never be able to fail |
| 27 | +// is left in the repo as a guard. No later CI run can expose it: CI builds |
| 28 | +// correctly, so the test is green there forever. Three independent sessions hit |
| 29 | +// this in one shift on three different packages; the one that caught it did so |
| 30 | +// by hand-grepping `dist/` for the mutation marker. This script is that grep, |
| 31 | +// mechanized, with the traps the manual version cannot see. |
| 32 | +// |
| 33 | +// ## The two ablation shapes, hence the two modes |
| 34 | +// |
| 35 | +// PLANT (default) the mutation ADDS something identifiable -- a changed |
| 36 | +// error code, a distinctive literal, a token in a message. |
| 37 | +// `<marker>` must be PRESENT in dist, or the run is void. |
| 38 | +// DELETE (--absent) the mutation REMOVES a guard. There is nothing to plant, |
| 39 | +// so the assertion inverts: a literal unique to the deleted |
| 40 | +// code must be GONE from dist. Same check, mirrored. |
| 41 | +// |
| 42 | +// `--absent` is also the restore leg: after putting the fix back and rebuilding, |
| 43 | +// it proves the marker really left the artifact. That leg matters more than it |
| 44 | +// looks -- a marker left behind in `dist/` keeps mutated code live for every |
| 45 | +// later suite run in that worktree, long after the ablation is "finished". |
| 46 | +// |
| 47 | +// ## Sourcemap-only matches are RED, not green |
| 48 | +// |
| 49 | +// A hit inside a `.map` file proves a sourcemap was regenerated, not that the |
| 50 | +// executable artifact carries the mutation. Counting it would rebuild the exact |
| 51 | +// false green this script exists to prevent, so `.map` hits are reported and |
| 52 | +// excluded from the verdict. |
| 53 | +// |
| 54 | +// ## Why this is not a `check:*` gate |
| 55 | +// |
| 56 | +// It judges a deliberately mutated working tree, so it can only be run by the |
| 57 | +// agent performing the ablation, at one specific moment between "mutate" and |
| 58 | +// "run the suite". CI has no ablation in flight and nothing to assert. It is |
| 59 | +// dev-side agent tooling, invoked from the ablation procedure in |
| 60 | +// `.claude/agents/os-dev.md` and `.claude/skills/dogfood-verification/SKILL.md` |
| 61 | +// -- keep those two and this file's usage line in step. |
| 62 | +// |
| 63 | +// Anything this script cannot see is RED, never a skip: a missing `dist/`, a |
| 64 | +// `dist/` with nothing readable in it, or a package name that resolves to |
| 65 | +// nothing all fail by name. A pre-flight that shrugs is worse than none, because |
| 66 | +// its exit 0 is read as proof. |
| 67 | + |
| 68 | +import { readFileSync, readdirSync, statSync, existsSync, mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs'; |
| 69 | +import { join, relative, resolve, extname } from 'node:path'; |
| 70 | +import { tmpdir } from 'node:os'; |
| 71 | +import { fileURLToPath } from 'node:url'; |
| 72 | +import process from 'node:process'; |
| 73 | + |
| 74 | +const REPO_ROOT = resolve(fileURLToPath(import.meta.url), '..', '..'); |
| 75 | + |
| 76 | +// Read as text, but never let a binary artifact fabricate a match. |
| 77 | +const BINARY_EXT = new Set(['.wasm', '.node', '.png', '.jpg', '.jpeg', '.gif', '.ico', '.woff', '.woff2', '.zip', '.gz', '.br']); |
| 78 | + |
| 79 | +/** Parse the `packages:` globs out of pnpm-workspace.yaml (no YAML dependency). */ |
| 80 | +export function parseWorkspaceGlobs(yamlText) { |
| 81 | + const globs = []; |
| 82 | + let inPackages = false; |
| 83 | + for (const rawLine of yamlText.split('\n')) { |
| 84 | + const line = rawLine.replace(/\s+$/, ''); |
| 85 | + if (/^packages:\s*$/.test(line)) { |
| 86 | + inPackages = true; |
| 87 | + continue; |
| 88 | + } |
| 89 | + if (inPackages) { |
| 90 | + const item = line.match(/^\s+-\s+['"]?([^'"\s]+)['"]?\s*$/); |
| 91 | + if (item) { |
| 92 | + globs.push(item[1]); |
| 93 | + continue; |
| 94 | + } |
| 95 | + if (line.trim() !== '') break; // next top-level key ends the list |
| 96 | + } |
| 97 | + } |
| 98 | + return globs; |
| 99 | +} |
| 100 | + |
| 101 | +/** name -> repo-relative dir, for every workspace package. */ |
| 102 | +function workspacePackages(repoRoot) { |
| 103 | + const yamlPath = join(repoRoot, 'pnpm-workspace.yaml'); |
| 104 | + let globs; |
| 105 | + try { |
| 106 | + globs = parseWorkspaceGlobs(readFileSync(yamlPath, 'utf8')); |
| 107 | + } catch { |
| 108 | + fail(`cannot read ${relative(repoRoot, yamlPath) || 'pnpm-workspace.yaml'} -- refusing to guess the workspace layout.`); |
| 109 | + } |
| 110 | + if (globs.length === 0) fail('pnpm-workspace.yaml declares no `packages:` globs -- refusing to scan an empty workspace.'); |
| 111 | + |
| 112 | + const dirs = []; |
| 113 | + for (const glob of globs) { |
| 114 | + if (glob.endsWith('/*')) { |
| 115 | + const parent = join(repoRoot, glob.slice(0, -2)); |
| 116 | + let entries = []; |
| 117 | + try { |
| 118 | + entries = readdirSync(parent, { withFileTypes: true }); |
| 119 | + } catch { |
| 120 | + continue; // a declared-but-absent parent is the workspace's problem, not ours |
| 121 | + } |
| 122 | + for (const e of entries) if (e.isDirectory()) dirs.push(join(parent, e.name)); |
| 123 | + } else { |
| 124 | + dirs.push(join(repoRoot, glob)); |
| 125 | + } |
| 126 | + } |
| 127 | + |
| 128 | + const byName = new Map(); |
| 129 | + for (const dir of dirs) { |
| 130 | + const pkgJson = join(dir, 'package.json'); |
| 131 | + if (!existsSync(pkgJson)) continue; |
| 132 | + try { |
| 133 | + const { name } = JSON.parse(readFileSync(pkgJson, 'utf8')); |
| 134 | + if (typeof name === 'string' && name.length > 0) byName.set(name, dir); |
| 135 | + } catch { |
| 136 | + // an unparseable package.json is not this script's verdict to give |
| 137 | + } |
| 138 | + } |
| 139 | + return byName; |
| 140 | +} |
| 141 | + |
| 142 | +/** Accept `@objectstack/plugin-auth`, `plugin-auth`, or a path to the package. */ |
| 143 | +export function resolvePackageDir(input, byName, repoRoot = REPO_ROOT) { |
| 144 | + if (byName.has(input)) return { dir: byName.get(input), name: input }; |
| 145 | + const scoped = `@objectstack/${input}`; |
| 146 | + if (byName.has(scoped)) return { dir: byName.get(scoped), name: scoped }; |
| 147 | + const asPath = resolve(repoRoot, input); |
| 148 | + for (const [name, dir] of byName) if (resolve(dir) === asPath) return { dir, name }; |
| 149 | + const near = [...byName.keys()].filter((n) => n.includes(input)).slice(0, 5); |
| 150 | + return { dir: null, name: null, near }; |
| 151 | +} |
| 152 | + |
| 153 | +function walkFiles(dir, out = []) { |
| 154 | + for (const e of readdirSync(dir, { withFileTypes: true })) { |
| 155 | + if (e.name === 'node_modules') continue; |
| 156 | + const full = join(dir, e.name); |
| 157 | + if (e.isDirectory()) walkFiles(full, out); |
| 158 | + else if (e.isFile()) out.push(full); |
| 159 | + } |
| 160 | + return out; |
| 161 | +} |
| 162 | + |
| 163 | +/** Scan a dist tree for `marker`, splitting code hits from sourcemap hits. */ |
| 164 | +export function scanDist(distDir, marker) { |
| 165 | + const codeHits = []; |
| 166 | + const mapHits = []; |
| 167 | + let scanned = 0; |
| 168 | + let skippedBinary = 0; |
| 169 | + for (const file of walkFiles(distDir)) { |
| 170 | + if (BINARY_EXT.has(extname(file))) { |
| 171 | + skippedBinary += 1; |
| 172 | + continue; |
| 173 | + } |
| 174 | + let buf; |
| 175 | + try { |
| 176 | + buf = readFileSync(file); |
| 177 | + } catch { |
| 178 | + continue; |
| 179 | + } |
| 180 | + if (buf.includes(0)) { |
| 181 | + skippedBinary += 1; // a NUL says binary, whatever the extension claims |
| 182 | + continue; |
| 183 | + } |
| 184 | + scanned += 1; |
| 185 | + if (buf.toString('utf8').includes(marker)) (file.endsWith('.map') ? mapHits : codeHits).push(file); |
| 186 | + } |
| 187 | + return { scanned, skippedBinary, codeHits, mapHits }; |
| 188 | +} |
| 189 | + |
| 190 | +/** |
| 191 | + * The whole verdict, as a pure function so the self-test can pin every branch. |
| 192 | + * mode: 'present' (plant ablation) | 'absent' (delete ablation / restore leg). |
| 193 | + */ |
| 194 | +export function verdict({ mode, distExists, scanned, codeHits, mapHits }) { |
| 195 | + if (!distExists) { |
| 196 | + return { ok: false, msg: 'no dist/ in this package -- it has never been built in this tree, so the suite consumes nothing you edited. Build it, then re-run this pre-flight.' }; |
| 197 | + } |
| 198 | + if (scanned === 0) { |
| 199 | + return { ok: false, msg: 'dist/ holds no readable text file -- refusing to call an empty scan a pass. Check the build actually produced output.' }; |
| 200 | + } |
| 201 | + if (mode === 'present') { |
| 202 | + if (codeHits > 0) { |
| 203 | + const extra = mapHits > 0 ? ` (plus ${mapHits} sourcemap hit${mapHits === 1 ? '' : 's'}, not counted)` : ''; |
| 204 | + return { ok: true, msg: `marker present in ${codeHits} built file${codeHits === 1 ? '' : 's'}${extra} -- the ablation is live in the artifact the suite consumes.` }; |
| 205 | + } |
| 206 | + if (mapHits > 0) { |
| 207 | + return { ok: false, msg: `marker found ONLY in ${mapHits} sourcemap file${mapHits === 1 ? '' : 's'} and in no executable output -- a sourcemap hit proves a rebuild happened somewhere, not that the running code carries the mutation. Treat this run as void.` }; |
| 208 | + } |
| 209 | + return { ok: false, msg: 'marker ABSENT from dist/ -- the suite would run the pre-mutation build and go GREEN on an ablation, certifying a test that may never be able to fail. Rebuild the package, then re-run this pre-flight.' }; |
| 210 | + } |
| 211 | + if (codeHits > 0) { |
| 212 | + return { ok: false, msg: `marker still present in ${codeHits} built file${codeHits === 1 ? '' : 's'} -- dist/ still carries the code you expected to be gone, so the run would test the wrong tree (and every later run in this worktree with it). Rebuild the package, then re-run this pre-flight.` }; |
| 213 | + } |
| 214 | + const extra = mapHits > 0 ? ` (${mapHits} stale sourcemap hit${mapHits === 1 ? '' : 's'} ignored -- sourcemaps do not execute)` : ''; |
| 215 | + return { ok: true, msg: `marker absent from all ${scanned} built files${extra} -- the artifact the suite consumes no longer carries it.` }; |
| 216 | +} |
| 217 | + |
| 218 | +function fail(msg) { |
| 219 | + console.error(`✗ ablation-dist-preflight: ${msg}`); |
| 220 | + process.exit(1); |
| 221 | +} |
| 222 | + |
| 223 | +function usage(msg) { |
| 224 | + console.error(`ablation-dist-preflight: ${msg}\n`); |
| 225 | + console.error(' node scripts/ablation-dist-preflight.mjs <package> <marker> marker MUST be in dist/ (planted ablation)'); |
| 226 | + console.error(' node scripts/ablation-dist-preflight.mjs <package> <marker> --absent marker must be GONE (deleted guard / restore leg)'); |
| 227 | + console.error(' node scripts/ablation-dist-preflight.mjs --self-test'); |
| 228 | + process.exit(2); |
| 229 | +} |
| 230 | + |
| 231 | +function run(argv) { |
| 232 | + const mode = argv.includes('--absent') ? 'absent' : 'present'; |
| 233 | + const positional = argv.filter((a) => !a.startsWith('--')); |
| 234 | + const [pkgArg, marker, ...rest] = positional; |
| 235 | + if (!pkgArg || !marker) usage('needs a package and a marker string.'); |
| 236 | + if (rest.length > 0) usage(`unexpected extra argument "${rest[0]}" -- quote the marker if it contains spaces.`); |
| 237 | + if (marker.trim().length === 0) usage('the marker is blank -- a blank marker matches everything and proves nothing.'); |
| 238 | + |
| 239 | + const byName = workspacePackages(REPO_ROOT); |
| 240 | + const { dir, name, near } = resolvePackageDir(pkgArg, byName); |
| 241 | + if (!dir) { |
| 242 | + const hint = near && near.length > 0 ? ` Did you mean: ${near.join(', ')}?` : ''; |
| 243 | + fail(`no workspace package matches "${pkgArg}".${hint}`); |
| 244 | + } |
| 245 | + |
| 246 | + const distDir = join(dir, 'dist'); |
| 247 | + const distExists = existsSync(distDir) && statSync(distDir).isDirectory(); |
| 248 | + const scan = distExists ? scanDist(distDir, marker) : { scanned: 0, skippedBinary: 0, codeHits: [], mapHits: [] }; |
| 249 | + const v = verdict({ mode, distExists, scanned: scan.scanned, codeHits: scan.codeHits.length, mapHits: scan.mapHits.length }); |
| 250 | + |
| 251 | + const where = relative(REPO_ROOT, distDir); |
| 252 | + console.log(`ablation-dist-preflight: ${name} -- ${mode === 'present' ? 'expecting' : 'expecting NO'} "${marker}" in ${where}`); |
| 253 | + for (const f of scan.codeHits.slice(0, 5)) console.log(` hit ${relative(REPO_ROOT, f)}`); |
| 254 | + if (scan.codeHits.length > 5) console.log(` hit ... and ${scan.codeHits.length - 5} more`); |
| 255 | + for (const f of scan.mapHits.slice(0, 3)) console.log(` map ${relative(REPO_ROOT, f)} (sourcemap, not counted)`); |
| 256 | + if (!v.ok) { |
| 257 | + console.error(`✗ ${v.msg}`); |
| 258 | + console.error(` rebuild: pnpm --filter ${name} build`); |
| 259 | + process.exit(1); |
| 260 | + } |
| 261 | + console.log(`✓ ${v.msg}`); |
| 262 | +} |
| 263 | + |
| 264 | +function selfTest() { |
| 265 | + const cases = [ |
| 266 | + ['missing dist is red', { mode: 'present', distExists: false, scanned: 0, codeHits: 0, mapHits: 0 }, false], |
| 267 | + ['empty dist is red, not a skip', { mode: 'present', distExists: true, scanned: 0, codeHits: 0, mapHits: 0 }, false], |
| 268 | + ['planted marker in code is green', { mode: 'present', distExists: true, scanned: 9, codeHits: 1, mapHits: 0 }, true], |
| 269 | + ['planted marker in code + map is green', { mode: 'present', distExists: true, scanned: 9, codeHits: 1, mapHits: 1 }, true], |
| 270 | + ['sourcemap-only hit is RED', { mode: 'present', distExists: true, scanned: 9, codeHits: 0, mapHits: 2 }, false], |
| 271 | + ['no hit at all is red', { mode: 'present', distExists: true, scanned: 9, codeHits: 0, mapHits: 0 }, false], |
| 272 | + ['absent mode: gone is green', { mode: 'absent', distExists: true, scanned: 9, codeHits: 0, mapHits: 0 }, true], |
| 273 | + ['absent mode: stale sourcemap tolerated', { mode: 'absent', distExists: true, scanned: 9, codeHits: 0, mapHits: 1 }, true], |
| 274 | + ['absent mode: still in code is red', { mode: 'absent', distExists: true, scanned: 9, codeHits: 3, mapHits: 0 }, false], |
| 275 | + ['absent mode: missing dist still red', { mode: 'absent', distExists: false, scanned: 0, codeHits: 0, mapHits: 0 }, false], |
| 276 | + ]; |
| 277 | + let failed = 0; |
| 278 | + for (const [label, input, expected] of cases) { |
| 279 | + const got = verdict(input).ok; |
| 280 | + if (got !== expected) { |
| 281 | + console.error(` ✗ ${label}: expected ok=${expected}, got ok=${got}`); |
| 282 | + failed += 1; |
| 283 | + } else { |
| 284 | + console.log(` ✓ ${label}`); |
| 285 | + } |
| 286 | + } |
| 287 | + |
| 288 | + // Filesystem leg: a real dist tree where the marker lives only in a sourcemap |
| 289 | + // is the trap the pure table above cannot exercise. |
| 290 | + const tmp = mkdtempSync(join(tmpdir(), 'ablation-preflight-')); |
| 291 | + try { |
| 292 | + const dist = join(tmp, 'dist'); |
| 293 | + mkdirSync(dist, { recursive: true }); |
| 294 | + writeFileSync(join(dist, 'index.js'), 'export const guard = () => "OS_ABLATION_TOKEN";\n'); |
| 295 | + writeFileSync(join(dist, 'other.js'), 'export const x = 1;\n'); |
| 296 | + writeFileSync(join(dist, 'other.js.map'), '{"sources":["OS_ONLY_IN_MAP"]}\n'); |
| 297 | + const planted = scanDist(dist, 'OS_ABLATION_TOKEN'); |
| 298 | + const mapOnly = scanDist(dist, 'OS_ONLY_IN_MAP'); |
| 299 | + const checks = [ |
| 300 | + ['scan finds the planted token in code', planted.codeHits.length === 1 && planted.mapHits.length === 0], |
| 301 | + ['scan classifies a map-only token as a map hit', mapOnly.codeHits.length === 0 && mapOnly.mapHits.length === 1], |
| 302 | + ['map-only scan is judged RED', verdict({ mode: 'present', distExists: true, scanned: mapOnly.scanned, codeHits: 0, mapHits: mapOnly.mapHits.length }).ok === false], |
| 303 | + ]; |
| 304 | + for (const [label, ok] of checks) { |
| 305 | + if (ok) console.log(` ✓ ${label}`); |
| 306 | + else { |
| 307 | + console.error(` ✗ ${label}`); |
| 308 | + failed += 1; |
| 309 | + } |
| 310 | + } |
| 311 | + } finally { |
| 312 | + rmSync(tmp, { recursive: true, force: true }); |
| 313 | + } |
| 314 | + |
| 315 | + if (failed > 0) { |
| 316 | + console.error(`✗ ablation-dist-preflight self-test: ${failed} case(s) failed.`); |
| 317 | + process.exit(1); |
| 318 | + } |
| 319 | + console.log('✓ ablation-dist-preflight self-test: all cases pass.'); |
| 320 | +} |
| 321 | + |
| 322 | +const argv = process.argv.slice(2); |
| 323 | +if (argv.includes('--self-test')) selfTest(); |
| 324 | +else run(argv); |
0 commit comments