From c3e3d95523eeb072712484713d9fd5fa96db6344 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 8 Sep 2026 16:43:31 +0000 Subject: [PATCH] test(coverage): make the README witness check capable of failing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `sourceFiles` collected every `.ts` under `src/`, tests included — and the witness table is a set of string literals in this very file. The corpus therefore contained the strings it was searching for, so every witness matched itself and the check could not fail for any reason. Three changes, each verified by breaking the tree and watching the right test go red rather than by reading the code: Test files are out of the corpus, and a self-check holds them out. Excluding them is one line, and without an assertion the next refactor of `sourceFiles` restores the loop with nothing failing. Reverting that line now fails with `expected [ …(836) ] to not include 'src/exampleCoverage.test.ts'`. It also asserts the corpus is non-empty, since an empty one passes everything below it. A witness must appear on an IMPORT, not anywhere in the file. `RateLimiter` occurs eleven times outside tests: once as a real `import type { RateLimiter } from "workglow"`, and ten times in prose or as a substring of a longer local name like `secFetchRateLimiterTableNames`. Ten of the eleven prove nothing and a substring search cannot tell them apart. The extraction strips comments first and matches whole statements rather than lines — these imports are routinely multi-line, and `createStandardKbStrategy` sits on a continuation line of one, which a line-wise filter reported as a missing package. The README's commands are resolved against the registered command tree, for the reason `commandsBoot.test.ts` gives: help text is prose, and a name appearing in some description satisfies a substring check without being registered. It walks the leading bare words of each backticked `sec …` and stops at the first flag or placeholder. Fifteen invocations resolve today; adding one for a command that does not exist fails it. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LWp6Z6wvAPDaDCjAFcTSj6 --- src/exampleCoverage.test.ts | 111 ++++++++++++++++++++++++++++++++++-- 1 file changed, 105 insertions(+), 6 deletions(-) diff --git a/src/exampleCoverage.test.ts b/src/exampleCoverage.test.ts index d7db772a..4bcaefe0 100644 --- a/src/exampleCoverage.test.ts +++ b/src/exampleCoverage.test.ts @@ -4,13 +4,23 @@ * SPDX-License-Identifier: Apache-2.0 */ +import { Command } from "commander"; import { readdirSync, readFileSync, statSync } from "node:fs"; -import { join } from "node:path"; +import { join, relative } from "node:path"; import { describe, expect, it } from "vitest"; +import { AddCommands } from "./commands/index"; const ROOT = join(import.meta.dirname, ".."); -/** Every `.ts` under `src/`, tests included. */ +/** + * Every non-test `.ts` under `src/`. + * + * Test files are excluded, and that exclusion is the whole point rather than a + * tidiness preference: this file is itself a `.ts` under `src/`, and the + * witness table below is a set of string literals in it. Including tests made + * the corpus contain the very strings it was searching for, so every witness + * matched itself and the check could not fail. + */ function sourceFiles(dir: string, out: string[] = []): string[] { for (const entry of readdirSync(dir)) { const full = join(dir, entry); @@ -19,11 +29,35 @@ function sourceFiles(dir: string, out: string[] = []): string[] { sourceFiles(full, out); continue; } - if (entry.endsWith(".ts")) out.push(full); + if (entry.endsWith(".ts") && !entry.endsWith(".test.ts")) out.push(full); } return out; } +/** + * The import statements in a file, which is where a witness has to appear. + * + * Matching anywhere in the file is what let prose stand in for evidence. + * `RateLimiter` occurs eleven times outside tests: once as a real + * `import type { RateLimiter } from "workglow"`, and ten times either inside a + * prose comment or as a substring of a longer local name like + * `secFetchRateLimiterTableNames`. Ten of those eleven prove nothing, and a + * substring check cannot tell them from the one that does. + */ +function importStatements(source: string): string[] { + // Comments first, so a witness named in prose cannot be read as an import. + const code = source.replace(/\/\*[\s\S]*?\*\//g, "").replace(/(^|[^:])\/\/.*$/gm, "$1"); + // Statement-wise, not line-wise: these imports are routinely multi-line, and + // `createStandardKbStrategy` sits on a continuation line of one. + return [ + ...code.matchAll(/\bimport\s+type\s+[\s\S]*?\bfrom\s*["'][^"']+["']/g), + ...code.matchAll(/\bimport\s+(?!type\b)[\s\S]*?\bfrom\s*["'][^"']+["']/g), + ...code.matchAll(/\bexport\s+[\s\S]*?\bfrom\s*["'][^"']+["']/g), + ...code.matchAll(/\bimport\s*\(\s*["'][^"']+["']\s*\)/g), + ...code.matchAll(/^\s*import\s*["'][^"']+["']/gm), + ].map((match) => match[0]); +} + /** * The README's "What does the work" table is a promise about which parts of the * library this example actually exercises. It is checked, because the promise @@ -33,7 +67,20 @@ function sourceFiles(dir: string, out: string[] = []): string[] { describe("the README's claims about what it demonstrates", () => { const readme = readFileSync(join(ROOT, "README.md"), "utf-8"); const sources = sourceFiles(join(ROOT, "src")); - const corpus = sources.map((file) => readFileSync(file, "utf-8")).join("\n"); + const importCorpus = sources + .flatMap((file) => importStatements(readFileSync(file, "utf-8"))) + .join("\n"); + + it("searches a corpus that does not contain this file", () => { + // The self-check. Excluding tests is one line in `sourceFiles`, and without + // an assertion holding it there the next refactor of that function restores + // the loop with nothing failing. + const relatives = sources.map((file) => relative(ROOT, file)); + expect(relatives).not.toContain(join("src", "exampleCoverage.test.ts")); + expect(relatives.filter((file) => file.endsWith(".test.ts"))).toEqual([]); + // And it still found the tree: an empty corpus would pass every check below. + expect(sources.length).toBeGreaterThan(100); + }); it("names a real path for every row of the table", () => { // Each row ends in one or more backticked paths under `src/`. @@ -53,7 +100,7 @@ describe("the README's claims about what it demonstrates", () => { // Reached through the `workglow` meta package, so a bare import of the // scoped name is not the evidence — the symbols are. One well-known export - // per package stands for it. + // per package stands for it, and has to appear on an import line. const witness: Readonly> = { "@workglow/job-queue": "RateLimiter", "@workglow/storage": "ITabularStorage", @@ -71,8 +118,60 @@ describe("the README's claims about what it demonstrates", () => { const symbol = witness[pkg]; // A package the README names and this test has no witness for is itself a // failure: the check is only worth anything if it covers the whole claim. - return symbol === undefined || !corpus.includes(symbol); + return symbol === undefined || !importCorpus.includes(symbol); }); expect(unproven).toEqual([]); }); }); + +/** + * The README prints commands for a reader to run. A command that does not exist + * costs that reader the first ten minutes of the example — which is the one + * stretch this repo exists to make work. + * + * Resolved against the registered tree rather than against `--help` text, for + * the reason `commandsBoot.test.ts` gives: help output is prose, and a group + * name that appears in some description satisfies a substring check without + * being registered at all. + */ +describe("the commands the README tells a reader to run", () => { + const readme = readFileSync(join(ROOT, "README.md"), "utf-8"); + + const program = new Command(); + AddCommands(program); + + /** Walks `sec a b c` down the command tree, or reports where it broke. */ + function resolve(words: readonly string[]): string | undefined { + let node: Command = program; + for (const word of words) { + const next: Command | undefined = node.commands.find( + (c) => c.name() === word || c.aliases().includes(word) + ); + if (next === undefined) return word; + node = next; + } + return undefined; + } + + it("resolves every backticked `sec …` against the command tree", () => { + const invocations = [...readme.matchAll(/`sec ([^`]+)`/g)].map((m) => m[1]!.trim()); + expect(invocations.length).toBeGreaterThan(5); + + const broken: string[] = []; + for (const invocation of new Set(invocations)) { + // Stop at the first thing that is not a bare word: a flag, a placeholder + // like , or a shell operator. Only the leading subcommand path is + // resolvable, and the rest is arguments. + const words: string[] = []; + for (const word of invocation.split(/\s+/)) { + if (!/^[a-z][a-z0-9-]*$/.test(word)) break; + words.push(word); + } + if (words.length === 0) continue; + const missing = resolve(words); + if (missing !== undefined) broken.push(`sec ${invocation} (no "${missing}")`); + } + + expect(broken).toEqual([]); + }); +});