fix(devx): read surface-2 docblocks from the shared comment scanner, not a lazy regex

packages/lint/scripts/check-doc-formula-expressions.mjs

@@ -152,6 +152,9 @@ import { join, relative, resolve, sep, dirname } from 'node:path';
 import { fileURLToPath } from 'node:url';
 
 import { requireDefaultExport, requireDependency } from '../../../scripts/import-prerequisite.mjs';
+// The tree's one comment/literal/code scanner. Surface 2's docblock extractor reads
+// its runs from here rather than from a block-comment regex — see {@link docblockSpans}.
+import { scanSource } from '../../../scripts/js-comment-mask.mjs';
 const ts = await requireDefaultExport('typescript', () => import('typescript'), import.meta.url);
 const {
   validateExpression,
@@ -1015,6 +1018,72 @@ function collectSpecFiles() {
   return out.sort();
 }
 
+/**
+ * The DOCBLOCK spans in a source, as `[start, end)` offsets — decided by
+ * `scripts/js-comment-mask.mjs`, this tree's one answer to "comment, literal, or
+ * code".
+ *
+ * ## Why this is not a regex (#12833)
+ *
+ * It was one: `/\/\*\*[\s\S]*?\*\//g`, an EXTRACTOR built out of the naive
+ * block-comment strip that `js-comment-mask.mjs`'s header exists to retire. A
+ * regex cannot see a string literal, so a `/**` sitting inside one opens a
+ * PHANTOM docblock that runs to the next real terminator, and `lastIndex` then
+ * skips every genuine docblock in between. The failure is silent and it is the
+ * bad direction: the gate reads FEWER `@example` bodies than the file holds and
+ * reports clean over prose it never looked at.
+ *
+ * Measured on `packages/spec/src` (1,061 `.ts`/`.tsx`, 13.7 MB) at
+ * `28a5c3e002`, which is why this is a fix rather than a tidy-up:
+ *
+ * - **9 files** where the regex's claimed docblock spans hold characters this
+ *   scanner does not call comment at all — 13,139 of them in
+ *   `kernel/manifest.test.ts`, where a glob string literal opens the phantom.
+ * - **9 files** (an OVERLAPPING BUT DIFFERENT set — 5 in common) where the
+ *   docblock COUNT moves.
+ * - **5 real docblocks** the regex was swallowing whole and this scan recovers.
+ *
+ * What did NOT move, stated because a fix nobody can see the effect of invites
+ * being undone: the `@example` body set is byte-identical either way, 416
+ * bodies, because none of the 5 recovered docblocks carries an `@example` and
+ * none of the phantoms fabricated one. So surface 2's "admits 0 sites today" is
+ * unchanged — but it is now a reading rather than the output of an extractor
+ * that provably could not see its own population. ⛔ Do not read the unchanged
+ * count as a reason to go back: the swallowed span is 13 KB wide and the next
+ * `@example` written behind one is invisible under the regex and judged here.
+ *
+ * ## What a "docblock run" is here
+ *
+ * `scanSource().comment` flags a comment's delimiters as well as its body, so a
+ * maximal run of flagged characters is one comment — except in two shapes the
+ * split below handles: block comments that ABUT (`/**a*\/\/**b*\/` is one run,
+ * two comments), and a run that opens with `//`, which is a line comment and
+ * never a docblock. An unterminated `/**` at EOF stays a docblock, matching what
+ * the TypeScript parser does with it; dropping it would be the silent direction.
+ */
+function docblockSpans(text) {
+  const { comment } = scanSource(text);
+  const spans = [];
+  const n = text.length;
+  let i = 0;
+  while (i < n) {
+    if (comment[i] !== 1) { i++; continue; }
+    let runEnd = i;
+    while (runEnd < n && comment[runEnd] === 1) runEnd++;
+    let k = i;
+    while (k + 1 < runEnd && text[k] === '/' && text[k + 1] === '*') {
+      const term = text.indexOf('*/', k + 2);
+      const end = term === -1 || term + 2 > runEnd ? runEnd : term + 2;
+      // `/**\/` is an empty block comment, not a docblock — and the regex this
+      // replaced did not match it either, so the parity is deliberate.
+      if (text.startsWith('/**', k) && end - k > 4) spans.push([k, end]);
+      k = end;
+    }
+    i = runEnd;
+  }
+  return spans;
+}
+
 /**
  * Every `@example` body in a file, as text, with the 1-based source line of its
  * first body line.
@@ -1027,16 +1096,17 @@ function collectSpecFiles() {
  * live. Pass A must see all 424; pass B needs the node (to know the slot) and so
  * is AST-bound by nature.
  *
+ * Text walk, but not a text SPLIT: which spans are docblocks comes from
+ * {@link docblockSpans} above, i.e. from the shared scanner.
+ *
  * The gutter strip is line-for-line, so a body line's index maps straight back to
  * a file line.
  */
 export function tsdocExampleBodies(text) {
   const out = [];
-  const re = /\/\*\*[\s\S]*?\*\//g;
-  let m;
-  while ((m = re.exec(text)) !== null) {
-    const commentStartLine = text.slice(0, m.index).split('\n').length; // 1-based
-    const raw = m[0].split('\n');
+  for (const [start, end] of docblockSpans(text)) {
+    const commentStartLine = text.slice(0, start).split('\n').length; // 1-based
+    const raw = text.slice(start, end).split('\n');
     const lines = raw.map((l, i) => {
       let s = l;
       if (i === 0) s = s.replace(/^\s*\/\*\*/, '');
@@ -1432,6 +1502,83 @@ const EXEMPTION_SELF_TEST_CASES = [
  * derived from ROOTS rather than re-spelled, so renaming or widening a root
  * cannot leave the declaration describing the old population.
  */
+/**
+ * The docblock EXTRACTOR's own both-directions test (#12833).
+ *
+ * {@link tsdocExampleBodies} used to split docblocks with a lazy block-comment
+ * regex, and the fixtures below are the two failure families
+ * `scripts/js-comment-mask.mjs`'s header names, reduced from shapes measured
+ * live in `packages/spec/src`:
+ *
+ * - **SWALLOWED** — a `/**` inside a glob string opens a phantom docblock whose
+ *   terminator is the NEXT real one, so the inline `@example` behind it lands on
+ *   the phantom's LAST line, the `*\/` strip runs instead of the `/**` strip, and
+ *   the tag no longer matches. The gate then reports clean over an example it
+ *   never read. `kernel/manifest.test.ts` holds a 13,139-character span of this.
+ * - **FABRICATED** — the same opener inside a string or template manufactures a
+ *   docblock that is not there, handing the gate prose to judge. This is the
+ *   over-count measured on `data/hook-body.zod.ts` and `ui/action.zod.ts`.
+ *
+ * The POSITIVE CONTROL is not decoration: every case above asserts a body count,
+ * and four of them assert a SMALLER one than the regex produced. Without a case
+ * that must come back non-empty, an extractor that returned nothing at all would
+ * pass this block.
+ */
+const EXTRACTOR_SELF_TEST_CASES = [
+  {
+    name: 'EXTRACTOR — POSITIVE CONTROL: an ordinary inline @example is extracted, at its own line',
+    holds: () => {
+      const found = tsdocExampleBodies(
+        'const A = 1;\nconst S = strictObject({\n'
+        + '  /** @example "status in [\'draft\']" - only drafts */\n  check: z.string(),\n});\n');
+      return found.length === 1 && found[0].startLine === 4;
+    },
+  },
+  {
+    name: 'EXTRACTOR — SWALLOWED: a glob string opens no docblock, so the @example behind it '
+      + 'is READ (the regex this replaced returned nothing here)',
+    holds: () => {
+      const found = tsdocExampleBodies(
+        "const GLOB = 'src/**/*.ts';\nconst S = strictObject({\n"
+        + '  /** @example "status in [\'draft\']" - only drafts */\n  check: z.string(),\n});\n');
+      return found.length === 1 && found[0].startLine === 4;
+    },
+  },
+  {
+    name: 'EXTRACTOR — FABRICATED: an @example quoted inside a string literal is prose, not a site',
+    holds: () => tsdocExampleBodies(
+      "const DOC = 'spell it /** @example \"a > 1\" */ above the field';\nexport const X = 1;\n",
+    ).length === 0,
+  },
+  {
+    name: 'EXTRACTOR — FABRICATED: ...and inside a TEMPLATE literal, where the phantom is not '
+      + 'line-bounded, only the real docblock survives',
+    holds: () => {
+      const found = tsdocExampleBodies(
+        'const T = `a /** @example "bogus" */ b`;\n/**\n * @example "record.a > 1"\n */\nconst X = 1;\n');
+      return found.length === 1 && found[0].startLine === 4;
+    },
+  },
+  {
+    name: 'EXTRACTOR — FABRICATED: the house-style note that spells `/** */` inside a `//` '
+      + 'comment is one line comment, not a second docblock',
+    holds: () => docblockSpans(
+      "// Declared with `//` (never `/** */`) and ABOVE the enum's JSDoc.\n"
+      + '/**\n * @example "record.a > 1"\n */\nconst X = 1;\n',
+    ).length === 1,
+  },
+  {
+    name: 'EXTRACTOR — two ABUTTING docblocks are two runs, not one (the shared scanner flags '
+      + 'them as a single contiguous comment span)',
+    holds: () => docblockSpans('/** @example "a > 1" *//** @example "b > 1" */\nconst X = 1;\n').length === 2,
+  },
+  {
+    name: 'EXTRACTOR — an empty block comment is not a docblock (parity with the regex this '
+      + 'replaced, which did not match it either)',
+    holds: () => docblockSpans('/**/\nconst X = 1;\n').length === 0,
+  },
+];
+
 const DECLARATION_SELF_TEST_CASES = [
   {
     name: 'DECLARATION — every ROOT the hint extractor cannot see is declared as a subtree '
@@ -1777,13 +1924,13 @@ function selfTest() {
   }
   failed += specSelfTest();
   failed += fieldRuleSelfTest();
-  for (const c of DECLARATION_SELF_TEST_CASES) {
+  for (const c of [...DECLARATION_SELF_TEST_CASES, ...EXTRACTOR_SELF_TEST_CASES]) {
     if (c.holds()) console.log(`  ✓ ${c.name}`);
     else { failed++; console.error(`  ✗ ${c.name}`); }
   }
   const total = SELF_TEST_CASES.length + SPEC_SELF_TEST_CASES.length + EXEMPTION_SELF_TEST_CASES.length
-    + DECLARATION_SELF_TEST_CASES.length + FIELD_RULE_SELF_TEST_CASES.length
-    + FIELD_RULE_REPORT_SELF_TEST_CASES.length;
+    + DECLARATION_SELF_TEST_CASES.length + EXTRACTOR_SELF_TEST_CASES.length
+    + FIELD_RULE_SELF_TEST_CASES.length + FIELD_RULE_REPORT_SELF_TEST_CASES.length;
   if (failed > 0) {
     console.error(`\n✗ check:doc-formula-expressions self-test: ${failed} case(s) failed`);
     process.exit(1);

scripts/check-comment-mask-adoption.mjs

@@ -135,8 +135,33 @@
  * extractor and the shared scanner disagree about how many docblocks 9 of the
  * 1,053 `packages/spec/src` sources contain. That is a row whose verdict
  * changes under the shared mask, which this header has always said is a finding
- * to read rather than something to absorb, so it is filed rather than fixed
- * here.
+ * to read rather than something to absorb, so it was filed rather than fixed
+ * here. The SECOND SHRINK below is that filing coming back.
+ *
+ * ## SECOND SHRINK (#12833): the one row whose verdict moved
+ *
+ * `packages/lint/scripts/check-doc-formula-expressions.mjs` is converted and its
+ * row is deleted here, in the same PR, which is the half the `stale` branch
+ * exists to demand. `tsdocExampleBodies()` now takes its docblock runs from
+ * `scanSource().comment` instead of a lazy `/**`-to-terminator regex, so the
+ * three literal forms that regex is blind to no longer open phantom docblocks.
+ *
+ * What the conversion found, re-measured on `28a5c3e002` (the population has
+ * grown to 1,061 files / 13.7 MB since the SECOND ROUND read it): both of that
+ * row's numbers reproduce exactly, the two sets of 9 files still overlap in only
+ * 5, and the scan recovers 5 real docblocks the regex was swallowing whole.
+ *
+ * And a third number the row could not have predicted: the gate's own
+ * `@example` body set does NOT move -- 416 bodies, byte for byte identical
+ * either way -- because none of the 5 recovered docblocks carries an `@example`
+ * and none of the phantoms fabricated one. The measuring round's expectation
+ * was that the count would go UP and that the new bodies would be the
+ * interesting part of the diff; on this tree it does not, and the conversion is
+ * verdict-preserving like the other nine. That is not a reason to leave the
+ * extractor alone, and it is recorded here so nobody re-derives the row from
+ * the unchanged count: "surface 2 admits 0 sites today" was the output of an
+ * extractor provably blind to 13 KB of its own population, and it is a reading
+ * now. Both directions are pinned in that gate's own `--self-test`.
  *
  * A recorded row that the scan no longer finds FAILS as stale. That is the
  * property `check-self-test-wired.mjs` names as the difference between a rule
@@ -270,8 +295,6 @@ const LEDGER = new Map([
     { shapes: ['regex-line'], verdict: 'unconverted', why: 'MEASURED #12475 over its real population (packages/create-objectstack/src/index.ts): deletes no live code -- it has no block arm at all -- and therefore KEEPS 2,744 chars of block-comment prose, the FABRICATION direction. Its line arm is anchored and reaches only whole-line comments' }],
   ['packages/drivers/driver-sql/src/live-dialect-matrix.isolation.test.ts',
     { shapes: ['regex-block', 'regex-line'], verdict: 'unconverted', why: 'MEASURED #12475 over its real population (codeOf() across all 152 *.test.ts in packages/drivers/driver-sql/src, 2.0 MB): 54 files disagree. Mostly the safe direction -- 4,582 chars of trailing line-comment prose KEPT by the anchored line arm -- but the block arm also DELETES 228 chars of live code in logger-receiver-detach.test.ts, where a fixture STRING quotes a docblock and the naive block regex eats the string. Its own direct-OS_TEST_URL offender set is empty under both strippers' }],
-  ['packages/lint/scripts/check-doc-formula-expressions.mjs',
-    { shapes: ['regex-block'], verdict: 'unconverted', why: 'MEASURED #12475 over its real population (collectSpecFiles(), 1,053 .ts/.tsx under packages/spec/src, 13.4 MB): tsdocExampleBodies() is an EXTRACTOR rather than a stripper, so it was measured as one -- do the spans it claims are docblocks hold characters the shared scanner calls code? On 9 files they do, up to 13,139 chars in kernel/manifest.test.ts, where a glob string opens a phantom docblock that then swallows the real ones behind it. THE ONLY ROW WHOSE OWN VERDICT MOVES: its docblock COUNT differs from the shared scanner\'s on 9 files too -- an OVERLAPPING BUT DIFFERENT set of 9, not the same one -- so this gate is under-reading @example bodies it believes it read. Filed rather than fixed here, per this header' }],
   ['packages/lint/src/validate-expressions.test.ts',
     { shapes: ['regex-block', 'regex-line'], verdict: 'unconverted', why: 'MEASURED #12475 over its real population (packages/lint/src/validate-expressions.ts, 75,403 chars): agrees with the shared mask. The unanchored trailing line arm is the dangerous spelling and it holds here only because that one rule source happens to carry no doubled slash inside a regex literal or string -- a property of today\'s file, not of this stripper' }],
   ['packages/lint/src/validate-org-axis-red-lines.test.ts',