From ebdcedc2f8c1e9ca2d8ab88ed8ade979d267852c Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 25 Sep 2026 04:07:28 -0400 Subject: [PATCH 1/6] fix(comparison): retire the alpha holdout split, and never fetch a withdrawn image MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cutout-wordmark-aflac, one of the eight alpha holdout images, was deleted from Wikimedia Commons on 2026-08-25 as a copyright violation. The pin table now records it as withdrawn, so it is never fetched, and asking for the alpha holdout split throws before anything is fetched: scoring the other seven would be a new experiment under the old one's name, and the split has already informed the decisions §11.12 records. The entry and its declared split stay. A cached copy with no entry would fall through splitFor to tune and join every alpha tune sweep. Refs #83 --- tools/comparison/src/alpha-images.ts | 36 +++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/tools/comparison/src/alpha-images.ts b/tools/comparison/src/alpha-images.ts index ebc896c0..a11be6a3 100644 --- a/tools/comparison/src/alpha-images.ts +++ b/tools/comparison/src/alpha-images.ts @@ -31,8 +31,32 @@ export interface AlphaImageSpec { softAlphaFraction: number; /** SHA-256 of the exact bytes (see corpus-pin.ts). */ sha256: string; + /** + * Set when the upstream source no longer exists: when and why it went. A + * withdrawn image is never fetched. Its entry stays so the pin still says + * what the results that scored it measured. It also keeps the image's + * declared split: a cached copy with no entry would fall through `splitFor` + * to "tune" and join every alpha tune sweep. + */ + withdrawn?: string; } +/** + * Why the alpha holdout split is closed. `ensureAlphaImages("holdout")` throws + * this rather than fetching a split that no longer exists as pinned. + * + * The split was eight images. One is gone, so scoring the other seven would be + * a different experiment reported under the old one's name. It has also been + * spent: §11.12 read it to adopt the alpha row, and to adopt the compact alpha + * row, and to reject `alpha_ac_fit`. A replacement is a new split, curated on + * covariates and sealed before it is read, not a re-run of this one. + */ +export const ALPHA_HOLDOUT_RETIRED = + "the alpha holdout split is retired (#83): cutout-wordmark-aflac, one of its eight images, " + + "was deleted from Wikimedia Commons on 2026-08-25 as a copyright violation and has no archived copy, " + + "and the split has already informed the decisions spec/EXPERIMENTS.md §11.12 records. " + + "Scoring the remaining seven is a new experiment, not a reproduction; a replacement needs a new, sealed split."; + /** * Curated alpha corpus, sourced from Wikimedia Commons under free licences * (see `fixtures/alpha/LICENSES.md` for per-image attribution). @@ -305,6 +329,8 @@ export const ALPHA_IMAGES: AlphaImageSpec[] = [ nonOpaqueFraction: 0.5971, softAlphaFraction: 0.0077, sha256: "3e0ddc1b3b6856b16b48ac2003f343e0e8cfaff02f022e8b2c21fbf24de2e302", + withdrawn: + "deleted from Wikimedia Commons on 2026-08-25 as a copyright violation (COM:CSD#F1); no archived copy (#83)", }, ]; @@ -315,17 +341,21 @@ export const ALPHA_IMAGES: AlphaImageSpec[] = [ * @param split Fetch only this split's images. A tune sweep never reads the * holdout images, and requiring them made every alpha sweep depend on the * one holdout file Wikimedia Commons has since deleted - * (`cutout-wordmark-aflac`, removed 2026-08-25 as a copyright violation): - * the tune split stays reproducible, and a holdout run still fails loudly - * rather than scoring a smaller corpus. + * (`cutout-wordmark-aflac`, removed 2026-08-25 as a copyright violation). + * The holdout split is retired, so asking for it throws + * {@link ALPHA_HOLDOUT_RETIRED} before anything is fetched, rather than + * failing on the missing file or scoring a smaller corpus. + * With no split, every image that has not been withdrawn is fetched. */ export async function ensureAlphaImages( split?: CorpusSplit, ): Promise { + if (split === "holdout") throw new Error(ALPHA_HOLDOUT_RETIRED); await fs.mkdir(ALPHA_DIR, { recursive: true }); const paths: string[] = []; let downloaded = 0; for (const spec of ALPHA_IMAGES) { + if (spec.withdrawn !== undefined) continue; if (split !== undefined && spec.split !== split) continue; const filePath = path.join(ALPHA_DIR, `${spec.label}${spec.ext}`); if ( From 342adadd2b075ed549e2b2a3f0c4bde010352b0b Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 25 Sep 2026 04:09:42 -0400 Subject: [PATCH 2/6] docs(spec): mark every claim resting on the alpha holdout as no longer reproducible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §11.12's alpha holdout column, the per-metric figures under it, and the alpha_ac_fit verdict were all read from a split that is now retired, so nothing can re-run them. Say so where each is stated: §11.12 at the table, §11.3 where the adopted row and alpha_ac_fit lean on it, §11.11 where the tier-1 row inherits it, and §11.13's evidence column. §11.12's provenance note no longer tells a reviewer to re-run a command that now refuses. The column's unbound note in verify-experiments names the retirement and what would lift it. Refs #83 --- spec/EXPERIMENTS.md | 54 ++++++++++++++++------ tools/comparison/src/verify-experiments.ts | 8 ++-- 2 files changed, 46 insertions(+), 16 deletions(-) diff --git a/spec/EXPERIMENTS.md b/spec/EXPERIMENTS.md index 2fe315d5..a03a7632 100644 --- a/spec/EXPERIMENTS.md +++ b/spec/EXPERIMENTS.md @@ -2361,6 +2361,12 @@ Not a corpus artifact, checked two ways. **Every one of the 16 images improves** including those only 15–22% transparent. And §11.2's control shows the direction belongs to alpha mode rather than to cut-out content. +Everything in this section is tune evidence. Its out-of-sample confirmation +was §11.12's alpha holdout, which is **retired and can no longer be +reproduced**: one of its images was deleted from Wikimedia Commons (§11.12, +#83). The adopted row rests on this section until a new alpha split is sealed +and read. + **The alpha channel also ran a generation behind the others.** L, a and b go through `quantize_ac_channel`, where `scale_fit` and `ac_nearest` live; alpha used a bare per-coefficient quantize against a nominal scale code, the v0.6 @@ -2373,7 +2379,9 @@ The knob has to be read against a fixed layout. The row −0.94 pp of that is the layout and only −0.22 pp is the knob: quoting the combined figure would overstate it fivefold. Small, free and principled — the same argument that adopted `ac_nearest` at 0.05% — which is exactly why -§11.12's holdout result gets to overrule it. +§11.12's holdout result got to overrule it. That result is from the retired +alpha split and cannot be re-run, so the default stands on a verdict nothing can +now check. ### 11.11 Does the alpha finding hold above tier 0? @@ -2411,6 +2419,10 @@ tier 1 — 0.06% behind `A24@3 L22@4 C5@3` — so **one row still serves tiers 1–3** and the `4^(tier−1)` structure is intact. Alpha AC counts are now 16 / 28 / 112 / 448 / 1792 across compact / 0 / 1 / 2 / 3 — monotone, which `validate.py` asserts. +This is a tune measurement only. The tier-1 row was never scored on a holdout, +and the tier-0 holdout figure it would lean on is from the retired alpha split, +which can no longer be reproduced (§11.12). + ### 11.12 The holdout, consulted once Every decision above was taken on tune, with holdout untouched until they were @@ -2454,6 +2466,22 @@ unchanged; the reason it is rejected is now over-determined. | A28@3 + `alpha_ac_fit` | — | −14.48% (worse than without) | rejected | | compact alpha A16@3 L12@4 C1@3 | −13.00% | −9.49%, guards ok | adopted | +> **This table's `holdout` column can no longer be reproduced, and nor can any +> figure below taken from the alpha holdout.** One of its eight images, +> `cutout-wordmark-aflac`, was deleted from Wikimedia Commons on 2026-08-25 as +> a copyright violation, and no archived copy was found. The split is +> **retired** (#83): `mise run sweep v07-holdout-alpha --split holdout` now +> refuses to run rather than fail on the missing file, and the column has no +> committed result (`verify:experiments --list-unbound` gives the reason). +> Scoring the remaining seven would be a new experiment reported under this +> one's name, and the split has already been read for every verdict in this +> table. So the figures stand as what was measured in `bd6a530`, and nothing +> can check them now. The `tune` column is bound and reproducible from +> `tools/comparison/results/v07-holdout-alpha.json`. The four decisions this +> table records keep their tune evidence (that column, and §11.3 for the alpha +> row and `alpha_ac_fit`) and have lost their out-of-sample evidence. They get it back only from a new alpha split, +> curated on covariates and sealed before anything reads it. +> > **The last row is measured against the 21-byte shipped shape**, not against > the 32-byte incumbent the three rows above it use — a 21 B candidate against a > 32 B baseline is a byte-count comparison wearing an allocation comparison's @@ -2467,19 +2495,19 @@ unchanged; the reason it is rejected is now over-determined. > **The moved cell's provenance**, because nothing automatic can supply it: both > the pinning and the figures in this table are one re-run of `mise run sweep > v07-holdout-alpha --split holdout` (§6) in `bd6a530`, against the Wikimedia -> corpus of `85f6af3` (§9.5). That output lands in the gitignored -> `output/sweeps/`, so the only check that holds this row to it is -> `verify:experiments`, which needs the sweep on disk and therefore does not run -> in CI — it reports a missing output as SKIP (§6). A reviewer re-deriving -> −9.49% runs that command on that corpus; a reviewer who has not is taking the -> cell on the commit that produced it, which is why the commit is named. +> corpus of `85f6af3` (§9.5). That output landed in the gitignored +> `output/sweeps/` and was never committed, and the split it scored is now +> retired (above), so the run cannot be repeated. Every holdout cell in this +> table rests on the commit that produced it, which is why the commit is named. -The alpha allocation validates emphatically: SSIMULACRA2 −307.2 → −242.7, -Butteraugli 57.56 → 44.36, DSSIM 0.2319 → 0.2179, αMAE 0.2696 → 0.1675. +The alpha allocation validated emphatically on that split: SSIMULACRA2 +−307.2 → −242.7, Butteraugli 57.56 → 44.36, DSSIM 0.2319 → 0.2179, αMAE +0.2696 → 0.1675. Those figures are from the retired split and cannot be +reproduced either. `alpha_ac_fit` does not, and is left at `false`. It is principled and free, and -it measured −0.21% on tune — but out of sample it is +0.09% alone and makes the -adopted allocation *worse* in combination. The knob stays for a future +it measured −0.21% on tune — but on the retired split it was +0.09% alone and +made the adopted allocation *worse* in combination. The knob stays for a future measurement; the default does not move on a result that will not replicate. **The compact tier's positioning claim, on holdout:** @@ -2496,7 +2524,7 @@ Beaten on all four metrics, out of sample, at ThumbHash's own size — the claim | Change | Evidence | |---|---| -| Alpha row → `L 22 @ 4, a/b 3 @ 3, A 28 @ 3` at tier 0 and the tier-1..3 base (the compact tier takes its own, below) | −16.19% holdout, all guards (§11.3, §11.11, §11.12) | +| Alpha row → `L 22 @ 4, a/b 3 @ 3, A 28 @ 3` at tier 0 and the tier-1..3 base (the compact tier takes its own, below) | −17.10% tune (§11.3, §11.11); −16.19% holdout, all guards, on the retired alpha split, which can no longer be reproduced (§11.12) | | Compact tier, code 4, 21 B, `L 19 @ 4 / a/b 6 @ 3` (alpha `L 12 @ 4 / a/b 1 @ 3 / A 16 @ 3`) | Beats ThumbHash on all four on holdout (§11.10, §11.12) | | Deadzone made reachable again | It was byte-identical at every value (§11.7) | @@ -2511,7 +2539,7 @@ Deliberately unchanged, each with the number that left it alone: | Quantization ranges | Every arm within ±0.17% (§11.8) | | No scalefactor bands | −0.13%, below threshold and unable to pay its signalling (§11.9) | | Tier-0 opaque layout | Holds on non-photographic content; no candidate significantly better (§11.4) | -| `alpha_ac_fit = false` | −0.21% on tune, +0.09% on holdout (§11.12) | +| `alpha_ac_fit = false` | −0.21% on tune, +0.09% on the retired alpha holdout, which can no longer be reproduced (§11.12) | What is still not measured, and is now the honest list for v0.8: diff --git a/tools/comparison/src/verify-experiments.ts b/tools/comparison/src/verify-experiments.ts index e14397c7..dbf35369 100644 --- a/tools/comparison/src/verify-experiments.ts +++ b/tools/comparison/src/verify-experiments.ts @@ -1729,8 +1729,10 @@ const BINDINGS: Binding[] = [ rowBaselines: ALPHA_HOLDOUT_ROW_BASELINES, }, // The `holdout` column was bound to v07-holdout-alpha-holdout. That run can - // no longer be recorded -- see UNBOUND_COLUMN_NOTES["11.12#1"] -- so the - // binding would be a SKIP forever. It is disclosed there instead. + // no longer be recorded: the alpha holdout split is retired + // (ALPHA_HOLDOUT_RETIRED in alpha-images.ts), so the binding would be a SKIP + // forever. It is disclosed in UNBOUND_COLUMN_NOTES["11.12#1"] instead, and + // §11.12 marks the column in the document itself. // §12 — the synthesis window, with the artifact columns that decide it. These // are the first bindings to check `meanRinging`/`meanSpurious`, which is the @@ -1952,7 +1954,7 @@ const UNBOUND_COLUMN_NOTES: Record = { "11.10#1": "ranks, derived by ordering two other sweeps' results rather than read from either", "11.12#1": - "`verdict` is the section's conclusion in words, not a measurement. `holdout` is UNREPRODUCIBLE: one of the alpha holdout images, cutout-wordmark-aflac, was deleted from Wikimedia Commons on 2026-08-25 as a copyright violation and has no archived copy, so v07-holdout-alpha --split holdout cannot be re-run and has no committed result (#83)", + "`verdict` is the section's conclusion in words, not a measurement. `holdout` is UNREPRODUCIBLE: one of the alpha holdout images, cutout-wordmark-aflac, was deleted from Wikimedia Commons on 2026-08-25 as a copyright violation and has no archived copy, so the alpha holdout split is retired, v07-holdout-alpha --split holdout refuses to run, and the column has no committed result. It stays unbound until a new sealed alpha split replaces it (#83)", "13.1#1": "`Spur / Deficit` is the ratio of two columns in the same row, both of which are bound; it is the reading, not a measurement", }; From 54086ccdb31c8cb3d6ba6a70604ce1fe66417756 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 25 Sep 2026 04:11:38 -0400 Subject: [PATCH 3/6] feat(comparison): probe every pinned Commons source's availability and licence corpus:licenses --check compares the attribution file with the pin table, both in this repository, so it could not notice Commons deleting cutout-wordmark-aflac as a copyright violation. --probe asks Commons instead: every photographic, alpha and graphics pin must still exist at its pinned URL, and every licence the pin table records must still be the one Commons records. Withdrawn entries are skipped and listed. Exit 1 when a source no longer holds, 2 when Commons could not be asked. It needs the network and a third-party host, so it is not a CI gate. Refs #83 --- .mise.toml | 8 +- tools/comparison/src/corpus-licenses.ts | 226 ++++++++++++++++++++++++ 2 files changed, 233 insertions(+), 1 deletion(-) diff --git a/.mise.toml b/.mise.toml index 63329047..72fdde6a 100644 --- a/.mise.toml +++ b/.mise.toml @@ -739,8 +739,14 @@ run = "node tools/comparison/dist/verify-experiments.js" # it did not exist until now, so the generator was reachable only as a bare # `node dist/corpus-licenses.js` and nothing ran `--check` anywhere. # --check fail if LICENSES.md is stale rather than rewriting it +# --probe ask Wikimedia Commons whether every pinned photographic, alpha and +# graphics source still exists at its pinned URL, and whether each +# recorded licence is still the one Commons records. Exit 1 when +# one no longer holds, 2 when Commons could not be asked. It needs +# the network, so CI does not run it (#83: a holdout image was +# deleted as a copyright violation, and --check could not notice). [tasks."corpus:licenses"] -description = "Regenerate the curated corpus attribution file from the pin table" +description = "Regenerate the curated corpus attribution file from the pin table, or --probe the pinned sources" depends = ["build:compare"] run = "node tools/comparison/dist/corpus-licenses.js" diff --git a/tools/comparison/src/corpus-licenses.ts b/tools/comparison/src/corpus-licenses.ts index 706ccb33..1ded4e27 100644 --- a/tools/comparison/src/corpus-licenses.ts +++ b/tools/comparison/src/corpus-licenses.ts @@ -14,10 +14,27 @@ * * mise run corpus:licenses # rewrite the file * mise run corpus:licenses --check # fail if it is stale (CI) + * mise run corpus:licenses --probe # fail if a pinned source is gone or relicensed + * + * `--check` compares two files in this repository, so it cannot notice the + * world changing under them. `cutout-wordmark-aflac` showed the cost. Commons + * deleted it as a copyright violation, and the attribution kept calling it + * freely licensed. Nothing noticed until a holdout run could not fetch it (#83). + * `--probe` asks the source instead. It covers the three Commons pin tables: + * photographic, alpha and graphics. Every entry must still exist on Commons at + * the pinned URL. Where the table records a licence (the photographic one + * does), Commons must still record the same one. An entry marked `withdrawn` is + * skipped and counted, since its absence is already known. Kodak24 is not + * probed: it is not on Commons, and its URLs are built inside + * `ensureHoldoutImages` rather than held in a table. The probe needs the network + * and a third-party host, so it is not a CI gate. Run it before a sweep that + * fetches the corpus, and when curating a new split. */ import fs from "node:fs/promises"; import path from "node:path"; +import { ALPHA_IMAGES } from "./alpha-images.ts"; +import { GRAPHIC_IMAGES } from "./graphic-images.ts"; import { CURATED_IMAGES } from "./natural-images.ts"; const OUT = path.resolve( @@ -75,7 +92,216 @@ function render(): string { return parts.join("\n"); } +/** One pinned Commons file, as the probe sees it. */ +interface ProbeTarget { + table: "natural" | "alpha" | "graphic"; + label: string; + url: string; + /** The licence the pin table records, when it records one. */ + licence?: string; +} + +const COMMONS_API = "https://commons.wikimedia.org/w/api.php"; +const PROBE_USER_AGENT = + "chromahash-corpus-probe/0.7 (https://github.com/visualcommons/chromahash) node-fetch"; +/** Commons answers at most 50 titles per query for an anonymous client. */ +const TITLES_PER_QUERY = 50; +const UPLOAD_URL = + /^https:\/\/upload\.wikimedia\.org\/wikipedia\/commons\/[0-9a-f]\/[0-9a-f]{2}\/([^/?#]+)$/; + +/** The part of an `action=query&prop=imageinfo` answer the probe reads. */ +interface CommonsQuery { + query?: { + normalized?: { from: string; to: string }[]; + redirects?: { from: string; to: string }[]; + pages?: Record< + string, + { + title: string; + missing?: string; + imageinfo?: { + url: string; + extmetadata?: { LicenseShortName?: { value: string } }; + }[]; + } + >; + }; +} + +/** `File:Name with spaces.png` for a pinned upload URL, or null when it is not one. */ +function commonsTitle(url: string): string | null { + const m = UPLOAD_URL.exec(url); + if (m?.[1] === undefined) return null; + return `File:${decodeURIComponent(m[1]).replace(/_/g, " ")}`; +} + +function sameUpload(a: string, b: string): boolean { + const strip = (u: string): string => decodeURIComponent(u.split("?")[0] ?? u); + return strip(a) === strip(b); +} + +function sameLicence(a: string, b: string): boolean { + const norm = (s: string): string => + s.trim().toLowerCase().replace(/\s+/g, " "); + return norm(a) === norm(b); +} + +function probeTargets(): { targets: ProbeTarget[]; withdrawn: string[] } { + const targets: ProbeTarget[] = [ + ...CURATED_IMAGES.map( + (s): ProbeTarget => ({ + table: "natural", + label: s.label, + url: s.urls[0] ?? "", + licence: s.licence, + }), + ), + ...GRAPHIC_IMAGES.map( + (s): ProbeTarget => ({ table: "graphic", label: s.label, url: s.url }), + ), + ]; + const withdrawn: string[] = []; + for (const s of ALPHA_IMAGES) { + if (s.withdrawn !== undefined) { + withdrawn.push(`${s.label}: ${s.withdrawn}`); + continue; + } + targets.push({ table: "alpha", label: s.label, url: s.url }); + } + return { targets, withdrawn }; +} + +/** + * Ask Commons about every pinned file and return what no longer holds, one + * line per problem. Throws when Commons cannot be asked at all. That is a + * failed probe, not a clean one. + */ +async function probe(targets: ProbeTarget[]): Promise { + const problems: string[] = []; + const byTitle = new Map(); + for (const t of targets) { + const title = commonsTitle(t.url); + if (title === null) { + problems.push( + `${t.table}/${t.label}: ${t.url} is not a Commons upload URL`, + ); + continue; + } + byTitle.set(title, t); + } + + const titles = [...byTitle.keys()]; + const seen = new Set(); + for (let i = 0; i < titles.length; i += TITLES_PER_QUERY) { + const batch = titles.slice(i, i + TITLES_PER_QUERY); + const params = new URLSearchParams({ + action: "query", + format: "json", + formatversion: "1", + prop: "imageinfo", + iiprop: "url|extmetadata", + iiextmetadatafilter: "LicenseShortName", + redirects: "1", + titles: batch.join("|"), + }); + const response = await fetch(`${COMMONS_API}?${params}`, { + headers: { "User-Agent": PROBE_USER_AGENT }, + }); + if (!response.ok) { + throw new Error( + `Commons answered HTTP ${response.status} to the probe; nothing was checked`, + ); + } + const body = (await response.json()) as CommonsQuery; + const pages = body.query?.pages; + if (pages === undefined) { + throw new Error("Commons returned no pages; nothing was checked"); + } + // Map each answer back to the title that was asked for. Commons rewrites + // a title it normalizes, and follows a redirect to the file's new name. + const asked = new Map(); + for (const t of batch) asked.set(t, t); + for (const { from, to } of body.query?.normalized ?? []) { + asked.set(to, asked.get(from) ?? from); + } + const redirected = new Map(); + for (const { from, to } of body.query?.redirects ?? []) { + const original = asked.get(from) ?? from; + asked.set(to, original); + redirected.set(original, to); + } + + for (const page of Object.values(pages)) { + const title = asked.get(page.title) ?? page.title; + const t = byTitle.get(title); + if (t === undefined) continue; + seen.add(title); + const where = `${t.table}/${t.label}`; + const info = page.imageinfo?.[0]; + if (page.missing !== undefined || info === undefined) { + problems.push( + `${where}: ${title} no longer exists on Commons (deleted or never uploaded)`, + ); + continue; + } + const moved = redirected.get(title); + if (moved !== undefined || !sameUpload(info.url, t.url)) { + problems.push( + `${where}: ${title} now resolves to ${moved ?? page.title} at ${info.url.split("?")[0]}, not the pinned ${t.url}`, + ); + } + const licence = info.extmetadata?.LicenseShortName?.value; + if (t.licence !== undefined) { + if (licence === undefined) { + problems.push(`${where}: Commons records no licence for ${title}`); + } else if (!sameLicence(licence, t.licence)) { + problems.push( + `${where}: Commons records "${licence}" for ${title}, the pin table "${t.licence}"`, + ); + } + } + } + } + for (const [title, t] of byTitle) { + if (!seen.has(title)) { + problems.push( + `${t.table}/${t.label}: Commons did not answer for ${title}`, + ); + } + } + return problems; +} + +async function runProbe(): Promise { + const { targets, withdrawn } = probeTargets(); + let problems: string[]; + try { + problems = await probe(targets); + } catch (e) { + console.error( + `The probe could not run: ${e instanceof Error ? e.message : String(e)}`, + ); + process.exit(2); + } + for (const w of withdrawn) console.log(`withdrawn, not probed: ${w}`); + if (problems.length > 0) { + for (const p of problems) console.error(p); + console.error( + `${problems.length} of ${targets.length} pinned Commons source(s) no longer hold as pinned.`, + ); + process.exit(1); + } + const licensed = targets.filter((t) => t.licence !== undefined).length; + console.log( + `All ${targets.length} pinned Commons sources exist at their pinned URL; ${licensed} recorded licence(s) match Commons.`, + ); +} + async function main(): Promise { + if (process.argv.includes("--probe")) { + await runProbe(); + return; + } const wanted = render(); const check = process.argv.includes("--check"); From 962f5741353ad83030872efe7cc522372ac20890 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 25 Sep 2026 04:16:12 -0400 Subject: [PATCH 4/6] =?UTF-8?q?docs(spec):=20give=20=C2=A711.13=20the=20al?= =?UTF-8?q?pha=20row's=20tier-1=20tune=20figure,=20and=20reflow=20=C2=A711?= =?UTF-8?q?.12's=20note?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §11.13 cited §11.11 for the −17.10% tune figure, which is §11.3's; §11.11 measures −11.95% at tier 1. Refs #83 --- spec/EXPERIMENTS.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/spec/EXPERIMENTS.md b/spec/EXPERIMENTS.md index a03a7632..34805b86 100644 --- a/spec/EXPERIMENTS.md +++ b/spec/EXPERIMENTS.md @@ -2479,8 +2479,9 @@ unchanged; the reason it is rejected is now over-determined. > can check them now. The `tune` column is bound and reproducible from > `tools/comparison/results/v07-holdout-alpha.json`. The four decisions this > table records keep their tune evidence (that column, and §11.3 for the alpha -> row and `alpha_ac_fit`) and have lost their out-of-sample evidence. They get it back only from a new alpha split, -> curated on covariates and sealed before anything reads it. +> row and `alpha_ac_fit`) and have lost their out-of-sample evidence. They get +> it back only from a new alpha split, curated on covariates and sealed before +> anything reads it. > > **The last row is measured against the 21-byte shipped shape**, not against > the 32-byte incumbent the three rows above it use — a 21 B candidate against a @@ -2524,7 +2525,7 @@ Beaten on all four metrics, out of sample, at ThumbHash's own size — the claim | Change | Evidence | |---|---| -| Alpha row → `L 22 @ 4, a/b 3 @ 3, A 28 @ 3` at tier 0 and the tier-1..3 base (the compact tier takes its own, below) | −17.10% tune (§11.3, §11.11); −16.19% holdout, all guards, on the retired alpha split, which can no longer be reproduced (§11.12) | +| Alpha row → `L 22 @ 4, a/b 3 @ 3, A 28 @ 3` at tier 0 and the tier-1..3 base (the compact tier takes its own, below) | −17.10% on tune (§11.3), −11.95% on tune at tier 1 (§11.11); −16.19% holdout, all guards, on the retired alpha split, which can no longer be reproduced (§11.12) | | Compact tier, code 4, 21 B, `L 19 @ 4 / a/b 6 @ 3` (alpha `L 12 @ 4 / a/b 1 @ 3 / A 16 @ 3`) | Beats ThumbHash on all four on holdout (§11.10, §11.12) | | Deadzone made reachable again | It was byte-identical at every value (§11.7) | From 8fe0addfdbc4323e2bb9ed7c3136db04991b8132 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 25 Sep 2026 04:22:13 -0400 Subject: [PATCH 5/6] =?UTF-8?q?docs(spec):=20split=20=C2=A711.13's=20compa?= =?UTF-8?q?ct-tier=20evidence,=20and=20mark=20the=20alpha=20layout's=20hol?= =?UTF-8?q?dout=20as=20the=20retired=20split?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- spec/EXPERIMENTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/EXPERIMENTS.md b/spec/EXPERIMENTS.md index 34805b86..36a6c8d3 100644 --- a/spec/EXPERIMENTS.md +++ b/spec/EXPERIMENTS.md @@ -2526,7 +2526,7 @@ Beaten on all four metrics, out of sample, at ThumbHash's own size — the claim | Change | Evidence | |---|---| | Alpha row → `L 22 @ 4, a/b 3 @ 3, A 28 @ 3` at tier 0 and the tier-1..3 base (the compact tier takes its own, below) | −17.10% on tune (§11.3), −11.95% on tune at tier 1 (§11.11); −16.19% holdout, all guards, on the retired alpha split, which can no longer be reproduced (§11.12) | -| Compact tier, code 4, 21 B, `L 19 @ 4 / a/b 6 @ 3` (alpha `L 12 @ 4 / a/b 1 @ 3 / A 16 @ 3`) | Beats ThumbHash on all four on holdout (§11.10, §11.12) | +| Compact tier, code 4, 21 B, `L 19 @ 4 / a/b 6 @ 3` (alpha `L 12 @ 4 / a/b 1 @ 3 / A 16 @ 3`) | Opaque layout: beats ThumbHash on all four on the photographic holdout (§11.10, §11.12). Alpha layout: −13.00% on tune against the 21 B shipped shape; −9.49% holdout, on the retired alpha split, which can no longer be reproduced (§11.12) | | Deadzone made reachable again | It was byte-identical at every value (§11.7) | Deliberately unchanged, each with the number that left it alone: From 90d41db95e1fb3775bc7a831a10a35d3247e77b3 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 25 Sep 2026 04:23:02 -0400 Subject: [PATCH 6/6] test(comparison): self-test the alpha holdout refusal and the withdrawn-pin skip --- tools/comparison/src/alpha-images.ts | 24 ++++++-- tools/comparison/src/metric-selftest.ts | 76 +++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 4 deletions(-) diff --git a/tools/comparison/src/alpha-images.ts b/tools/comparison/src/alpha-images.ts index a11be6a3..db3856cc 100644 --- a/tools/comparison/src/alpha-images.ts +++ b/tools/comparison/src/alpha-images.ts @@ -334,6 +334,24 @@ export const ALPHA_IMAGES: AlphaImageSpec[] = [ }, ]; +/** + * The pins {@link ensureAlphaImages} fetches for `split`: withdrawn entries + * are skipped, and the retired holdout split throws + * {@link ALPHA_HOLDOUT_RETIRED}. Separate from the fetch so the self-test + * (`metric-selftest.ts`) can assert both without the network. + */ +export function alphaImagesToFetch( + split?: CorpusSplit, + images: readonly AlphaImageSpec[] = ALPHA_IMAGES, +): AlphaImageSpec[] { + if (split === "holdout") throw new Error(ALPHA_HOLDOUT_RETIRED); + return images.filter( + (spec) => + spec.withdrawn === undefined && + (split === undefined || spec.split === split), + ); +} + /** * Ensure every alpha fixture is present and content-pinned. A fetch failure or * digest mismatch throws — see `ensureNaturalImages` for why. @@ -350,13 +368,11 @@ export const ALPHA_IMAGES: AlphaImageSpec[] = [ export async function ensureAlphaImages( split?: CorpusSplit, ): Promise { - if (split === "holdout") throw new Error(ALPHA_HOLDOUT_RETIRED); + const specs = alphaImagesToFetch(split); await fs.mkdir(ALPHA_DIR, { recursive: true }); const paths: string[] = []; let downloaded = 0; - for (const spec of ALPHA_IMAGES) { - if (spec.withdrawn !== undefined) continue; - if (split !== undefined && spec.split !== split) continue; + for (const spec of specs) { const filePath = path.join(ALPHA_DIR, `${spec.label}${spec.ext}`); if ( await ensurePinnedFixture({ diff --git a/tools/comparison/src/metric-selftest.ts b/tools/comparison/src/metric-selftest.ts index 0533e13a..171f29b9 100644 --- a/tools/comparison/src/metric-selftest.ts +++ b/tools/comparison/src/metric-selftest.ts @@ -50,9 +50,19 @@ * register (`experiments-register.ts`) and the result reader's shape check * (`results.ts`) fail a run only on a state the committed document and results * are never in, so each failure branch is driven from a fixture instead. + * + * The alpha corpus block covers the retired alpha holdout split and withdrawn + * pins (`alpha-images.ts`, #83), whose refusal and skip no sweep CI runs ever + * reaches. */ import { spawnSync } from "node:child_process"; +import { + ALPHA_HOLDOUT_RETIRED, + type AlphaImageSpec, + alphaImagesToFetch, +} from "./alpha-images.ts"; +import { splitFor } from "./corpus.ts"; import { computeRinging } from "./metrics/local.ts"; import { computeSpurious } from "./metrics/spurious.ts"; import { aspectFidelity, log2ToPct } from "./aspect.ts"; @@ -2133,6 +2143,72 @@ console.log("\nverify:experiments — the table register and result shape\n"); ); } +// --- Alpha corpus: the retired holdout and withdrawn pins (#83) ------------ +// +// `ensureAlphaImages` refuses the retired holdout split and never fetches a +// withdrawn pin. Neither branch runs in any sweep CI executes, so both are +// driven here through the selection it delegates to, without the network. +{ + console.log("\nalpha corpus selection:"); + + let thrown = ""; + try { + alphaImagesToFetch("holdout"); + } catch (e) { + thrown = e instanceof Error ? e.message : String(e); + } + check( + "the retired alpha holdout split is refused", + thrown === ALPHA_HOLDOUT_RETIRED, + thrown === "" ? "no error thrown" : thrown, + ); + + const base: AlphaImageSpec = { + label: "fixture-kept", + url: "https://example.invalid/kept.png", + ext: ".png", + width: 1, + height: 1, + split: "tune", + nonOpaqueFraction: 0, + softAlphaFraction: 0, + sha256: "0".repeat(64), + }; + const fixture: AlphaImageSpec[] = [ + base, + { ...base, label: "fixture-withdrawn", withdrawn: "gone" }, + { ...base, label: "fixture-holdout", split: "holdout" }, + ]; + const labels = (specs: AlphaImageSpec[]): string => + specs.map((s) => s.label).join(","); + const tune = labels(alphaImagesToFetch("tune", fixture)); + check( + "a withdrawn tune pin is not fetched for the tune split", + tune === "fixture-kept", + tune, + ); + const all = labels(alphaImagesToFetch(undefined, fixture)); + check( + "a withdrawn pin is not fetched with no split", + all === "fixture-kept,fixture-holdout", + all, + ); + + const shipped = alphaImagesToFetch().filter( + (s) => s.withdrawn !== undefined || s.label === "cutout-wordmark-aflac", + ); + check( + "the deleted cutout-wordmark-aflac pin is never fetched", + shipped.length === 0, + labels(shipped), + ); + check( + "the withdrawn pin keeps its holdout split, so a cached copy cannot join tune", + splitFor("cutout-wordmark-aflac") === "holdout", + splitFor("cutout-wordmark-aflac"), + ); +} + console.log( failures === 0 ? "\nAll metric self-checks passed.\n"