ci: red the Test Core shard when its predicted time stops matching the measured one

.github/workflows/ci.yml

@@ -539,7 +539,7 @@ jobs:
           node scripts/partition-test-shards.mjs "$RUNNER_TEMP/turbo-ls.json" \
             --shard ${{ matrix.shard }}/6 --exclude @objectstack/dogfood \
             > "$RUNNER_TEMP/shard-packages.txt"
-          echo "Packages on this shard:"
+          echo 'Items on this shard (a package name, or a package plus a k/n file-level slice):'
           cat "$RUNNER_TEMP/shard-packages.txt"
 
       # --concurrency=4: turbo's default (10) oversubscribes the 4-vCPU
@@ -587,11 +587,69 @@ jobs:
           # vitest's own "use the default" signal. Needs turbo.json's
           # globalPassThroughEnv entry or turbo strips it — see the script header.
           export VITEST_MAX_WORKERS="$(node scripts/vitest-worker-cap.mjs)"
-          FILTERS=$(sed 's/^/--filter=/' "$RUNNER_TEMP/shard-packages.txt" | tr '\n' ' ')
           mkdir -p "$RUNNER_TEMP/stall-reports"
-          node scripts/run-with-stall-guard.mjs --log "$RUNNER_TEMP/test-core.log" --stall-minutes 10 \
-            --report-dir "$RUNNER_TEMP/stall-reports" -- \
-            pnpm turbo run test $FILTERS --concurrency=4 --summarize --log-order=stream
+
+          # Split the shard's ITEMS into the whole packages, which share one
+          # turbo run as they always have, and the file-level slices, which
+          # cannot: `--shard=k/n` is passed through to vitest by turbo as a
+          # RUN-level argument, so it would reach every package in the run —
+          # and on any package with fewer test files than n that is a hard
+          # vitest failure (or, with --passWithNoTests, silently no tests at
+          # all). A slice therefore gets its own invocation, filtered to the one
+          # package the partitioner sliced.
+          FILTERS=""
+          SLICES=""
+          while read -r PKG SLICE; do
+            [ -n "$PKG" ] || continue
+            if [ -n "$SLICE" ]; then
+              SLICES="$SLICES $PKG=$SLICE"
+            else
+              FILTERS="$FILTERS --filter=$PKG"
+            fi
+          done < "$RUNNER_TEMP/shard-packages.txt"
+
+          # Each leg tees to its OWN log — run-with-stall-guard opens the log
+          # with 'w', so a second leg pointed at one path would truncate the
+          # first leg's output and the completeness guard below would grade half
+          # a shard. They are concatenated afterwards, and that concatenation
+          # happens even when a leg failed, because a red suite is exactly when
+          # the completeness guard earns its keep.
+          #
+          # ⛔ A failing leg STOPS the remaining ones, the same way turbo stops
+          # scheduling on the first failure inside one run. Carrying on would add
+          # a second full suite to a job that is already red and already inside a
+          # 30-minute wall — turning an informative red into a killed job with no
+          # attestation at all, which is the #16173 failure mode itself.
+          STATUS=0
+          LOGS=""
+          for LEG in __whole__ $SLICES; do
+            if [ "$LEG" = __whole__ ]; then
+              [ -n "$FILTERS" ] || continue
+              LOG="$RUNNER_TEMP/test-core-packages.log"
+              set -- pnpm turbo run test $FILTERS --concurrency=4 --summarize --log-order=stream
+            else
+              PKG="${LEG%%=*}"
+              SLICE="${LEG#*=}"
+              LOG="$RUNNER_TEMP/test-core-slice-$(printf '%s' "$PKG" | tr -c 'A-Za-z0-9' '-').log"
+              set -- pnpm turbo run test "--filter=$PKG" --concurrency=4 --summarize --log-order=stream -- "--shard=$SLICE"
+            fi
+            LOGS="$LOGS $LOG"
+            node scripts/run-with-stall-guard.mjs --log "$LOG" --stall-minutes 10 \
+              --report-dir "$RUNNER_TEMP/stall-reports" -- "$@" || { STATUS=$?; break; }
+          done
+
+          # Only over logs that EXIST, and via an explicit `if` rather than a
+          # `&&` chain: a leg whose guard never got far enough to open its log
+          # would otherwise make `cat` non-zero, and under `set -e` that replaces
+          # the SUITE's exit status with cat's — the step would report the wrong
+          # reason for its own red.
+          : > "$RUNNER_TEMP/test-core.log"
+          for LOG in $LOGS; do
+            if [ -f "$LOG" ]; then
+              cat "$LOG" >> "$RUNNER_TEMP/test-core.log"
+            fi
+          done
+          exit $STATUS
 
       # --summarize above costs nothing at runtime and writes
       # `.turbo/runs/<id>.json`: one per-task record with the execution window
@@ -635,6 +693,31 @@ jobs:
           if-no-files-found: ignore
           retention-days: 1
 
+      # ⛔ THE DRIFT STEP IS DELIBERATELY NOT WIRED HERE YET (#16173).
+      #
+      # partition-test-shards.mjs carries a fully-tested `--check-drift` mode —
+      # it reads the summary uploaded above back and reds when a shard's MEASURED
+      # test total outruns its PREDICTED one past MAX_MEASURED_OVER_PREDICTED. The
+      # code, its self-tests and its ablation are all on this branch; only this
+      # invocation waits, and the wait is a SEQUENCING decision, not an oversight.
+      #
+      # Why: scripts/test-shard-timings.json is still stale for @objectstack/cli
+      # (458.15s recorded, 1231.52s measured), so wiring the step today would red
+      # the shard carrying a CLI slice on every single PR — a true reading, but one
+      # that blocks everything until the dataset is refreshed. The file-level split
+      # above already removes the urgent hazard on its own, taking the worst shard
+      # from ~1445s (80% of this job's 30-minute wall) to ~1059s (59%) with the
+      # dataset untouched, because the CLI is halved across two runners instead of
+      # falling on one.
+      #
+      # ⇒ The refresh and this step land TOGETHER in the follow-up, in that order.
+      # The refresh recipe is in PR #16220's body. When it lands, restore a step
+      # here that runs `--check-drift` over `.turbo/runs/*.json` with
+      # `--label "Test Core (${{ matrix.shard }}/6)"`, with NO `if:` and NO
+      # `continue-on-error` (the point is the red), placed ABOVE the attestation
+      # pair for the #6082 reason documented on the upload above — so a drift red
+      # also withholds the attestation, which is the fail-closed direction.
+
       # Runs even when the suite failed — that is when it earns its keep. It
       # answers TWO questions about a red suite, and needs both to be able to
       # say anything at all about a green one.

scripts/check-test-completeness.mjs

@@ -144,6 +144,12 @@ import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 import process from 'node:process';
 import { isEntrypoint } from './invoked-as.mjs';
+// The shard-item grammar, from the script that WRITES the scheduled list. A
+// second reader here would be a second grammar, and the two would drift apart
+// silently -- an unparsed `@objectstack/cli 1/2` reaches describe() as a
+// package name the turbo ls document has never heard of, which this guard
+// (correctly, for its own contract) refuses the whole shard over.
+import { parseShardItem } from './partition-test-shards.mjs';
 
 const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
 
@@ -298,14 +304,49 @@ export function parseFailedTestPackages(text) {
   return failed;
 }
 
+/**
+ * The scheduled list, folded to PACKAGE names.
+ *
+ * `shard-packages.txt` holds shard ITEMS, and since #16173 an item can be a
+ * file-level slice of a package (`@objectstack/cli 1/2`). Q2 is asked per
+ * package -- did this package report at all on this shard -- and the reported
+ * set is keyed by the package name turbo prints, which carries no slice, so an
+ * unfolded item would be a name the `turbo ls` document does not list and the
+ * shard would be refused whole. Duplicates collapse for the same reason: a
+ * package is complete when every slice of it scheduled HERE has reported, and
+ * partition-test-shards.mjs refuses any split that puts two slices of one
+ * package on one shard, so "every slice here" is "the one slice here".
+ */
+export function scheduledPackages(lines) {
+  const out = [];
+  const seen = new Set();
+  for (const line of lines) {
+    const { name } = parseShardItem(line);
+    if (seen.has(name)) continue;
+    seen.add(name);
+    out.push(name);
+  }
+  return out;
+}
+
 // true = every task turbo scheduled succeeded, so nothing was cancelled.
 // false = it stopped early. null = turbo never printed its roster (it died, or
 // the log was truncated) -- unknown, and treated as "not completed" by Rule B.
+//
+// EVERY roster line is folded, not just the last one. A shard's log holds one
+// roster per `turbo run` invocation, and since #16173 a shard that carries a
+// file-level slice runs TWO (the sliced package needs its own passthrough, which
+// turbo applies run-wide). Reading only the last would let a completed second
+// invocation vouch for a first one that stopped early -- and Rule B turns that
+// into a red on every package the abort left unreached, which is precisely the
+// false-red machine this file's header warns about.
 export function parseRunCompleted(text) {
   let completed = null;
   for (const line of text.split('\n')) {
     const m = line.match(TASKS);
-    if (m) completed = Number(m[1]) === Number(m[2]);
+    if (!m) continue;
+    const thisRun = Number(m[1]) === Number(m[2]);
+    completed = completed === null ? thisRun : completed && thisRun;
   }
   return completed;
 }
@@ -596,7 +637,7 @@ function reportVerdict(verdict) {
 // not red. A battery BELOW its floor means cases stopped running; the remedy is
 // to find what stopped registering.
 const SELF_TEST_BATTERIES = Object.freeze({
-  'check-test-completeness self-test': 67,
+  'check-test-completeness self-test': 79,
 });
 
 // DELETING an entry silences that battery's floor exactly as effectively as
@@ -956,6 +997,63 @@ function selfTest({ quiet = false } = {}) {
     'exit codes: the refusal code collides with a verdict code',
   );
 
+  // -- File-level slice items in the scheduled list (#16173) ----------------
+  //
+  // `shard-packages.txt` stopped being a list of package names the day one
+  // package started being sharded below package granularity. Both halves below
+  // fail in the false-RED direction if they regress, which is the direction
+  // this file's header spends its length warning about.
+  eq(scheduledPackages(['@objectstack/spec', '@objectstack/core']), ['@objectstack/spec', '@objectstack/core'],
+    'scheduled: plain package names were disturbed');
+  eq(scheduledPackages(['@objectstack/cli 1/2']), ['@objectstack/cli'],
+    'scheduled: a slice did not fold to its package name');
+  eq(scheduledPackages(['@objectstack/cli 1/2', '@objectstack/cli 2/2']), ['@objectstack/cli'],
+    'scheduled: two slices of one package did not collapse');
+  eq(scheduledPackages(['@objectstack/spec', '@objectstack/cli 2/2', '@objectstack/core']),
+    ['@objectstack/spec', '@objectstack/cli', '@objectstack/core'],
+    'scheduled: folding reordered the list');
+  // END-TO-END: an unfolded item reaches describe() as a name the turbo ls
+  // document cannot resolve, and classifyShard refuses the whole shard over it.
+  // This asserts the fold is what stops that, not that describe() got lenient.
+  eq(
+    classifyShard({
+      scheduled: scheduledPackages(['@objectstack/cli 1/2']),
+      reported: new Set(['@objectstack/cli']),
+      failed: new Set(),
+      runCompleted: true,
+      describe: describe({ '@objectstack/cli': { hasTestScript: true, testFileCount: 268 } }),
+    }).silent,
+    [],
+    'scheduled: a slice that DID report was still graded silent',
+  );
+  eq(
+    threw(() =>
+      classifyShard({
+        scheduled: ['@objectstack/cli 1/2'],
+        reported: new Set(['@objectstack/cli']),
+        failed: new Set(),
+        runCompleted: true,
+        describe: describe({ '@objectstack/cli': { hasTestScript: true, testFileCount: 268 } }),
+      }),
+    ),
+    true,
+    'scheduled: an UNFOLDED item was accepted, so the fold is not what makes this work',
+  );
+
+  // -- One roster per turbo invocation, and a shard can now run two (#16173) --
+  const roster = (ok, total) => `Tasks:    ${ok} successful, ${total} total`;
+  eq(parseRunCompleted(roster(4, 4)), true, 'runCompleted: a complete single roster');
+  eq(parseRunCompleted(roster(3, 5)), false, 'runCompleted: an incomplete single roster');
+  eq(parseRunCompleted('no roster here'), null, 'runCompleted: a log with no roster is unknown, not complete');
+  eq(parseRunCompleted(`${roster(4, 4)}\n${roster(2, 2)}`), true,
+    'runCompleted: two complete rosters');
+  // The load-bearing one: the packages the FIRST invocation never reached would
+  // otherwise be charged as silent under Rule B.
+  eq(parseRunCompleted(`${roster(3, 9)}\n${roster(1, 1)}`), false,
+    'runCompleted: a completed second invocation vouched for a first that stopped early');
+  eq(parseRunCompleted(`${roster(9, 9)}\n${roster(0, 1)}`), false,
+    'runCompleted: a second invocation that stopped early was overlooked');
+
   // ⚠️ The floor is scoped to the LOUD run on purpose. `selfTest({ quiet: true })`
   // also runs on EVERY production invocation of this gate (see `main()`), and
   // there nothing claims a self-test verdict — the floor exists to stop a green
@@ -1059,10 +1157,12 @@ function main() {
   if (scheduledPath) {
     let scheduled;
     try {
-      scheduled = readFileSync(scheduledPath, 'utf8')
-        .split('\n')
-        .map((l) => l.trim())
-        .filter(Boolean);
+      scheduled = scheduledPackages(
+        readFileSync(scheduledPath, 'utf8')
+          .split('\n')
+          .map((l) => l.trim())
+          .filter(Boolean)
+      );
     } catch (err) {
       console.error(`check-test-completeness: cannot read ${scheduledPath} -- ${err.message}`);
       process.exit(1);