ci: target a four-minute full PR quality loop

.github/scripts/e2e-duration-history.mjs

@@ -0,0 +1,91 @@
+#!/usr/bin/env node
+
+import { readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
+import { basename, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const SCHEMA_VERSION = 1;
+const PREVIOUS_WEIGHT = 0.7;
+
+function timingFiles(root) {
+  const files = [];
+  for (const entry of readdirSync(root)) {
+    const candidate = join(root, entry);
+    if (statSync(candidate).isDirectory()) files.push(...timingFiles(candidate));
+    else if (entry.endsWith(".json")) files.push(candidate);
+  }
+  return files.sort();
+}
+function normalizeBase(value) {
+  if (typeof value === "number") return { durationMs: value, samples: 1 };
+  return {
+    durationMs: value?.durationMs,
+    samples: Number.isInteger(value?.samples) ? value.samples : 1,
+  };
+}
+
+export function mergeDurationHistory({ base, reports }) {
+  const tests = Object.fromEntries(
+    Object.entries(base?.tests ?? {}).map(([id, value]) => [id, normalizeBase(value)]),
+  );
+  const observed = new Set();
+  for (const { name, report } of reports) {
+    if (report?.schemaVersion !== SCHEMA_VERSION) throw new Error(`${name} has an unsupported schema`);
+    if (report.status !== "passed") throw new Error(`${name} did not record a clean Playwright run`);
+    for (const [id, timing] of Object.entries(report.tests ?? {})) {
+      if (observed.has(id)) throw new Error(`duplicate timing for test ${id}`);
+      observed.add(id);
+      if (timing.status !== "passed" || !Number.isFinite(timing.durationMs) || timing.durationMs <= 0) continue;
+      const previous = tests[id];
+      tests[id] = previous && Number.isFinite(previous.durationMs)
+        ? {
+            durationMs: Math.round(previous.durationMs * PREVIOUS_WEIGHT + timing.durationMs * (1 - PREVIOUS_WEIGHT)),
+            samples: Math.min(20, previous.samples + 1),
+          }
+        : { durationMs: Math.round(timing.durationMs), samples: 1 };
+    }
+  }
+  if (observed.size === 0) throw new Error("no passing test timings were found");
+  return {
+    schemaVersion: SCHEMA_VERSION,
+    algorithm: "ewma-0.7",
+    tests: Object.fromEntries(Object.entries(tests).sort(([left], [right]) => left.localeCompare(right))),
+  };
+}
+
+function parseArguments(argv) {
+  const values = {};
+  for (let index = 0; index < argv.length; index += 2) {
+    const key = argv[index];
+    const value = argv[index + 1];
+    if (!key?.startsWith("--") || value === undefined) throw new Error(`invalid argument: ${key ?? "<missing>"}`);
+    values[key.slice(2)] = value;
+  }
+  return values;
+}
+
+function main() {
+  const args = parseArguments(process.argv.slice(2));
+  if (!args.base || !args.input || !args.output) {
+    throw new Error("usage: e2e-duration-history.mjs --base <json> --input <dir> --output <json>");
+  }
+  const reports = timingFiles(args.input).map((file) => ({
+    name: basename(file),
+    report: JSON.parse(readFileSync(file, "utf8")),
+  }));
+  const result = mergeDurationHistory({
+    base: JSON.parse(readFileSync(args.base, "utf8")),
+    reports,
+  });
+  writeFileSync(args.output, `${JSON.stringify(result, null, 2)}\n`);
+  process.stdout.write(`updated ${Object.keys(result.tests).length} duration records from ${reports.length} shards\n`);
+}
+
+if (process.argv[1] === fileURLToPath(import.meta.url)) {
+  try {
+    main();
+  } catch (error) {
+    console.error(`e2e-duration-history: ${error.message}`);
+    process.exitCode = 1;
+  }
+}

.github/scripts/e2e-duration-history.test.mjs

@@ -0,0 +1,35 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { mergeDurationHistory } from "./e2e-duration-history.mjs";
+
+test("merges clean shard timings with a bounded moving average", () => {
+  const result = mergeDurationHistory({
+    base: { tests: { existing: 10_000, old: { durationMs: 4_000, samples: 20 } } },
+    reports: [
+      { name: "shard-1.json", report: { schemaVersion: 1, status: "passed", tests: {
+        existing: { durationMs: 20_000, status: "passed" },
+        unseen: { durationMs: 5_000, status: "passed" },
+        skipped: { durationMs: 0, status: "skipped" },
+      } } },
+      { name: "shard-2.json", report: { schemaVersion: 1, status: "passed", tests: {
+        old: { durationMs: 6_000, status: "passed" },
+      } } },
+    ],
+  });
+  assert.deepEqual(result.tests.existing, { durationMs: 13_000, samples: 2 });
+  assert.deepEqual(result.tests.unseen, { durationMs: 5_000, samples: 1 });
+  assert.deepEqual(result.tests.old, { durationMs: 4_600, samples: 20 });
+  assert.equal(result.tests.skipped, undefined);
+});
+test("rejects failed reports and duplicate coverage", () => {
+  assert.throws(() => mergeDurationHistory({
+    base: { tests: {} },
+    reports: [{ name: "failed.json", report: { schemaVersion: 1, status: "failed", tests: {} } }],
+  }), /did not record a clean/);
+  const report = { schemaVersion: 1, status: "passed", tests: { same: { durationMs: 1_000, status: "passed" } } };
+  assert.throws(() => mergeDurationHistory({
+    base: { tests: {} },
+    reports: [{ name: "one.json", report }, { name: "two.json", report }],
+  }), /duplicate timing/);
+});

.github/scripts/e2e-duration-plan.mjs

@@ -0,0 +1,159 @@
+#!/usr/bin/env node
+
+import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const SCHEMA_VERSION = 1;
+const DEFAULT_DURATION_MS = 8_000;
+const MINIMUM_DURATION_MS = 1_000;
+
+function walkSuites(suites, ancestors = [], tests = []) {
+  for (const suite of suites ?? []) {
+    const nextAncestors = suite.title && suite.title !== suite.file
+      ? [...ancestors, suite.title]
+      : ancestors;
+    for (const spec of suite.specs ?? []) {
+      for (const project of spec.tests ?? []) {
+        tests.push({
+          id: spec.id,
+          projectName: project.projectName,
+          file: spec.file,
+          line: spec.line,
+          column: spec.column,
+          path: nextAncestors.filter(Boolean),
+          title: spec.title,
+          tags: spec.tags ?? [],
+        });
+      }
+    }
+    walkSuites(suite.suites, nextAncestors, tests);
+  }
+  return tests;
+}
+
+export function inventoryTests(report, { project = "chromium", tag } = {}) {
+  const tests = walkSuites(report?.suites).filter((test) =>
+    test.projectName === project && (!tag || test.tags.includes(tag)),
+  );
+  const ids = new Set();
+  const selectors = new Set();
+  for (const test of tests) {
+    if (!test.id || ids.has(test.id)) {
+      throw new Error(`inventory contains a missing or duplicate test id: ${test.id ?? "<missing>"}`);
+    }
+    ids.add(test.id);
+    test.selector = [
+      `[${test.projectName}]`,
+      `${test.file}:${test.line}:${test.column}`,
+      ...test.path,
+      test.title,
+    ].join(" › ");
+    if (selectors.has(test.selector)) {
+      throw new Error(`inventory contains a duplicate selector: ${test.selector}`);
+    }
+    selectors.add(test.selector);
+  }
+  if (tests.length === 0) throw new Error("inventory did not contain any matching tests");
+  return tests;
+}
+
+function durationFor(test, history) {
+  const value = history?.tests?.[test.id];
+  const duration = typeof value === "number" ? value : value?.durationMs;
+  return Number.isFinite(duration) && duration > 0
+    ? Math.max(MINIMUM_DURATION_MS, Math.round(duration))
+    : DEFAULT_DURATION_MS;
+}
+
+export function buildDurationPlan({ tests, history, shardCount }) {
+  if (!Number.isInteger(shardCount) || shardCount < 1) {
+    throw new Error("shard count must be a positive integer");
+  }
+  const shards = Array.from({ length: shardCount }, (_, index) => ({
+    shard: index + 1,
+    predictedWorkMs: 0,
+    tests: [],
+  }));
+  const weighted = tests
+    .map((test) => ({ ...test, durationMs: durationFor(test, history) }))
+    .sort((left, right) => right.durationMs - left.durationMs || left.id.localeCompare(right.id));
+
+  for (const test of weighted) {
+    shards.sort((left, right) =>
+      left.predictedWorkMs - right.predictedWorkMs ||
+      left.tests.length - right.tests.length ||
+      left.shard - right.shard,
+    );
+    shards[0].tests.push(test);
+    shards[0].predictedWorkMs += test.durationMs;
+  }
+  shards.sort((left, right) => left.shard - right.shard);
+
+  const plannedIds = shards.flatMap((shard) => shard.tests.map((test) => test.id));
+  if (plannedIds.length !== tests.length || new Set(plannedIds).size !== tests.length) {
+    throw new Error("duration plan did not assign every inventory test exactly once");
+  }
+  return {
+    schemaVersion: SCHEMA_VERSION,
+    algorithm: "longest-predicted-first",
+    testCount: tests.length,
+    shardCount,
+    defaultDurationMs: DEFAULT_DURATION_MS,
+    shards,
+  };
+}
+
+export function writeDurationPlan(plan, outputDirectory) {
+  mkdirSync(outputDirectory, { recursive: true });
+  const manifest = {
+    ...plan,
+    shards: plan.shards.map((shard) => ({
+      shard: shard.shard,
+      predictedWorkMs: shard.predictedWorkMs,
+      testCount: shard.tests.length,
+      file: `shard-${shard.shard}.txt`,
+    })),
+  };
+  for (const shard of plan.shards) {
+    writeFileSync(
+      resolve(outputDirectory, `shard-${shard.shard}.txt`),
+      `${shard.tests.map((test) => test.selector).join("\n")}\n`,
+    );
+  }
+  writeFileSync(resolve(outputDirectory, "manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`);
+  return manifest;
+}
+
+function parseArguments(argv) {
+  const values = {};
+  for (let index = 0; index < argv.length; index += 2) {
+    const key = argv[index];
+    const value = argv[index + 1];
+    if (!key?.startsWith("--") || value === undefined) throw new Error(`invalid argument: ${key ?? "<missing>"}`);
+    values[key.slice(2)] = value;
+  }
+  return values;
+}
+
+function main() {
+  const args = parseArguments(process.argv.slice(2));
+  if (!args.inventory || !args.history || !args.output || !args.shards) {
+    throw new Error("usage: e2e-duration-plan.mjs --inventory <json> --history <json> --output <dir> --shards <count> [--tag <tag>]");
+  }
+  const inventory = JSON.parse(readFileSync(args.inventory, "utf8"));
+  const history = JSON.parse(readFileSync(args.history, "utf8"));
+  const tests = inventoryTests(inventory, { tag: args.tag });
+  const plan = buildDurationPlan({ tests, history, shardCount: Number(args.shards) });
+  const manifest = writeDurationPlan(plan, args.output);
+  process.stdout.write(`${JSON.stringify(manifest, null, 2)}\n`);
+}
+
+if (process.argv[1] === fileURLToPath(import.meta.url)) {
+  try {
+    main();
+  } catch (error) {
+    console.error(`e2e-duration-plan: ${error.message}`);
+    process.exitCode = 1;
+  }
+}

.github/scripts/e2e-duration-plan.test.mjs

@@ -0,0 +1,71 @@
+import assert from "node:assert/strict";
+import { mkdtempSync, readFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import test from "node:test";
+
+import { buildDurationPlan, inventoryTests, writeDurationPlan } from "./e2e-duration-plan.mjs";
+
+function inventory() {
+  return {
+    suites: [{
+      title: "lesson.spec.ts",
+      file: "lesson.spec.ts",
+      suites: [{
+        title: "lesson journey",
+        specs: [
+          { id: "slow", title: "slow path", file: "lesson.spec.ts", line: 10, column: 3, tags: ["lane:critical"], tests: [{ projectName: "chromium" }] },
+          { id: "medium", title: "medium path", file: "lesson.spec.ts", line: 20, column: 3, tags: [], tests: [{ projectName: "chromium" }] },
+          { id: "fast", title: "fast path", file: "lesson.spec.ts", line: 30, column: 3, tags: ["lane:critical"], tests: [{ projectName: "chromium" }] },
+          { id: "webkit", title: "other browser", file: "lesson.spec.ts", line: 40, column: 3, tags: [], tests: [{ projectName: "webkit" }] },
+        ],
+      }],
+    }],
+  };
+}
+
+test("balances measured durations and assigns every Chromium test once", () => {
+  const tests = inventoryTests(inventory());
+  const plan = buildDurationPlan({
+    tests,
+    history: { tests: { slow: 20_000, medium: 11_000, fast: 9_000 } },
+    shardCount: 2,
+  });
+  assert.equal(plan.testCount, 3);
+  assert.deepEqual(plan.shards.map((shard) => shard.predictedWorkMs), [20_000, 20_000]);
+  assert.deepEqual(
+    plan.shards.flatMap((shard) => shard.tests.map((candidate) => candidate.id)).sort(),
+    ["fast", "medium", "slow"],
+  );
+});
+
+test("uses a conservative default for unseen tests and can select a tagged lane", () => {
+  const tests = inventoryTests(inventory(), { tag: "lane:critical" });
+  const plan = buildDurationPlan({ tests, history: { tests: { slow: 12_000 } }, shardCount: 2 });
+  assert.deepEqual(plan.shards.map((shard) => shard.predictedWorkMs), [12_000, 8_000]);
+});
+
+test("writes Playwright test-list files plus a compact manifest", () => {
+  const output = mkdtempSync(join(tmpdir(), "e2e-duration-plan-"));
+  const plan = buildDurationPlan({
+    tests: inventoryTests(inventory()),
+    history: { tests: {} },
+    shardCount: 2,
+  });
+  const manifest = writeDurationPlan(plan, output);
+  assert.equal(manifest.testCount, 3);
+  const selectors = [1, 2]
+    .flatMap((shard) => readFileSync(join(output, `shard-${shard}.txt`), "utf8").trim().split("\n"))
+    .sort();
+  assert.deepEqual(selectors, [
+    "[chromium] › lesson.spec.ts:10:3 › lesson journey › slow path",
+    "[chromium] › lesson.spec.ts:20:3 › lesson journey › medium path",
+    "[chromium] › lesson.spec.ts:30:3 › lesson journey › fast path",
+  ]);
+});
+
+test("rejects duplicate ids instead of silently dropping coverage", () => {
+  const report = inventory();
+  report.suites[0].suites[0].specs[1].id = "slow";
+  assert.throws(() => inventoryTests(report), /duplicate test id/);
+});