ci(e2e): benchmark build reuse and worker count

.github/e2e-shard-capacity.json

@@ -33,5 +33,26 @@
     "minimumRelativeGain": 0.05,
     "requiresEveryShardToPass": true,
     "preservesFullChromiumSuite": true
+  },
+  "runtimeOptimization": {
+    "runId": 33397529873,
+    "url": "https://github.com/msrivas-7/CodeTutor-AI/actions/runs/33397529873",
+    "headSha": "60e20db6ae8686e33cb4331c993e79af8f5c7cbc",
+    "totalTests": 439,
+    "imageReuse": {
+      "localBuildEndToEndSeconds": 369,
+      "prebuiltEndToEndSecondsIncludingPreparation": 338,
+      "preparationSeconds": 24,
+      "absoluteGainSeconds": 31,
+      "relativeGain": 0.084,
+      "selected": true
+    },
+    "workerCandidates": [
+      { "workers": 2, "testCriticalPathSeconds": 190, "reliable": true, "selected": true },
+      { "workers": 3, "testCriticalPathSeconds": 160, "reliable": false, "selected": false },
+      { "workers": 4, "testCriticalPathSeconds": null, "reliable": false, "selected": false }
+    ],
+    "maximumChromiumShards": 20,
+    "method": "Three images prepared in parallel, then local-build and digest-pinned 16-shard stages ran sequentially on the same commit with zero retries. Higher worker counts were eligible only when every shard passed."
   }
 }

.github/scripts/e2e-forwarded-ip.mjs

@@ -0,0 +1,29 @@
+#!/usr/bin/env node
+
+import { createHash } from "node:crypto";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+/**
+ * Map a CI job namespace onto an address in 2001:db8::/32, the IPv6 prefix
+ * reserved for documentation. Each isolated Docker stack then exercises the real
+ * trusted-proxy and per-IP quota path without sharing one database counter.
+ */
+export function e2eForwardedIp(namespace) {
+  const normalized = String(namespace ?? "").trim();
+  if (!normalized || normalized.length > 240) {
+    throw new Error("E2E namespace must contain 1-240 characters");
+  }
+  const digest = createHash("sha256").update(`codetutor-e2e-client-v1:${normalized}`).digest();
+  const segments = Array.from({ length: 6 }, (_, index) =>
+    digest.readUInt16BE(index * 2).toString(16));
+  return `2001:db8:${segments.join(":")}`;
+}
+
+function main() {
+  const namespace = process.argv[2];
+  if (!namespace) throw new Error("Usage: e2e-forwarded-ip.mjs <job-namespace>");
+  console.log(e2eForwardedIp(namespace));
+}
+
+if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) main();

.github/scripts/e2e-forwarded-ip.test.mjs

@@ -0,0 +1,41 @@
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import test from "node:test";
+
+import { e2eForwardedIp } from "./e2e-forwarded-ip.mjs";
+
+const e2eWorkflow = readFileSync(new URL("../workflows/e2e.yml", import.meta.url), "utf8");
+const topologyWorkflow = readFileSync(new URL("../workflows/e2e-shard-topology.yml", import.meta.url), "utf8");
+const securityWorkflow = readFileSync(new URL("../workflows/security.yml", import.meta.url), "utf8");
+const compose = readFileSync(new URL("../../docker-compose.yml", import.meta.url), "utf8");
+const viteConfig = readFileSync(new URL("../../frontend/vite.config.ts", import.meta.url), "utf8");
+
+test("derives a stable address from the reserved benchmark range", () => {
+  const value = e2eForwardedIp("shard-5-run33390478931-attempt1");
+  assert.equal(value, e2eForwardedIp("shard-5-run33390478931-attempt1"));
+  assert.match(value, /^2001:db8(?::[0-9a-f]{1,4}){6}$/);
+});
+
+test("isolates shards, attempts, lanes, and benchmark stages", () => {
+  const namespaces = [
+    "shard-5-run33390478931-attempt1",
+    "shard-6-run33390478931-attempt1",
+    "shard-5-run33390478931-attempt2",
+    "cross-browser-webkit-run33390478931-attempt1",
+    "benchmark-prebuilt-w3-5-run33390478931-attempt1",
+  ];
+  assert.equal(new Set(namespaces.map(e2eForwardedIp)).size, namespaces.length);
+});
+
+test("rejects absent and unbounded namespaces", () => {
+  assert.throws(() => e2eForwardedIp(""), /1-240/);
+  assert.throws(() => e2eForwardedIp("x".repeat(241)), /1-240/);
+});
+
+test("every Compose-backed browser lane installs its isolated proxy identity", () => {
+  assert.equal((e2eWorkflow.match(/name: Allocate isolated anonymous client identity/g) ?? []).length, 3);
+  assert.equal((topologyWorkflow.match(/name: Allocate isolated anonymous client identity/g) ?? []).length, 1);
+  assert.equal((securityWorkflow.match(/name: Allocate isolated anonymous client identity/g) ?? []).length, 1);
+  assert.match(compose, /E2E_FORWARDED_FOR: "\$\{E2E_FORWARDED_FOR:-}"/);
+  assert.match(viteConfig, /"x-forwarded-for": e2eForwardedFor/);
+});

.github/scripts/e2e-runtime-benchmark.mjs

@@ -0,0 +1,220 @@
+#!/usr/bin/env node
+
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const SHARDS = 16;
+const MINIMUM_RELATIVE_GAIN = 0.05;
+const MINIMUM_ABSOLUTE_GAIN_SECONDS = 20;
+const BOOT_STEP_NAME = "Boot docker-compose stack";
+const TEST_STEP_NAME = "Run identical full suite without retries";
+const PREPARATION_JOBS = [
+  "Build backend E2E image once",
+  "Build runner E2E image once",
+  "Build frontend E2E image once",
+];
+const EXPERIMENTS = [
+  { id: "local-w2", imageMode: "local-build", workers: 2 },
+  { id: "prebuilt-w2", imageMode: "prebuilt", workers: 2 },
+  { id: "prebuilt-w3", imageMode: "prebuilt", workers: 3 },
+  { id: "prebuilt-w4", imageMode: "prebuilt", workers: 4 },
+];
+
+function secondsBetween(start, end) {
+  if (!start || !end) return null;
+  const value = (Date.parse(end) - Date.parse(start)) / 1000;
+  return Number.isFinite(value) && value >= 0 ? value : null;
+}
+
+function stepSeconds(job, stepName) {
+  const step = job.steps?.find((candidate) => candidate.name === stepName);
+  return step ? secondsBetween(step.started_at, step.completed_at) : null;
+}
+
+function percentile(values, fraction) {
+  if (values.length === 0) return null;
+  const sorted = [...values].sort((left, right) => left - right);
+  return sorted[Math.ceil(fraction * sorted.length) - 1];
+}
+
+function summarizeExperiment(experiment, run, jobs, totalTests) {
+  const prefix = `Playwright runtime ${experiment.id} (`;
+  const selected = jobs.filter((job) => job.name.includes(prefix));
+  const starts = selected.map((job) => Date.parse(job.started_at)).filter(Number.isFinite);
+  const topologyStartedAt = starts.length === SHARDS
+    ? new Date(Math.min(...starts)).toISOString()
+    : null;
+  const execution = selected
+    .map((job) => secondsBetween(job.started_at, job.completed_at))
+    .filter((value) => value !== null);
+  const relativeCompletion = selected
+    .map((job) => secondsBetween(topologyStartedAt, job.completed_at))
+    .filter((value) => value !== null);
+  const boots = selected.map((job) => stepSeconds(job, BOOT_STEP_NAME)).filter((value) => value !== null);
+  const tests = selected.map((job) => stepSeconds(job, TEST_STEP_NAME)).filter((value) => value !== null);
+  const reliable = selected.length === SHARDS
+    && selected.every((job) => job.conclusion === "success")
+    && boots.length === SHARDS
+    && tests.length === SHARDS;
+
+  return {
+    ...experiment,
+    runId: run.id,
+    shards: SHARDS,
+    expectedJobs: SHARDS,
+    observedJobs: selected.length,
+    reliable,
+    totalTests,
+    topologyStartedAt,
+    topologyReadySeconds: relativeCompletion.length === SHARDS
+      ? Math.max(...relativeCompletion)
+      : null,
+    slowestExecutionSeconds: execution.length === SHARDS ? Math.max(...execution) : null,
+    aggregateExecutionSeconds: execution.length === SHARDS
+      ? execution.reduce((sum, value) => sum + value, 0)
+      : null,
+    bootSeconds: {
+      median: boots.length === SHARDS ? percentile(boots, 0.5) : null,
+      p90: boots.length === SHARDS ? percentile(boots, 0.9) : null,
+      max: boots.length === SHARDS ? Math.max(...boots) : null,
+    },
+    testCriticalPathSeconds: tests.length === SHARDS ? Math.max(...tests) : null,
+    aggregateTestSeconds: tests.length === SHARDS
+      ? tests.reduce((sum, value) => sum + value, 0)
+      : null,
+    testImbalanceSeconds: tests.length === SHARDS
+      ? Math.max(...tests) - Math.min(...tests)
+      : null,
+    jobs: selected.map((job) => ({
+      name: job.name,
+      conclusion: job.conclusion ?? "unknown",
+      executionSeconds: secondsBetween(job.started_at, job.completed_at),
+      bootSeconds: stepSeconds(job, BOOT_STEP_NAME),
+      testSeconds: stepSeconds(job, TEST_STEP_NAME),
+    })),
+  };
+}
+
+function summarizePreparation(jobs) {
+  const selected = jobs.filter((job) =>
+    PREPARATION_JOBS.some((name) => job.name.includes(name)));
+  const starts = selected.map((job) => Date.parse(job.started_at)).filter(Number.isFinite);
+  const startedAt = starts.length === PREPARATION_JOBS.length
+    ? new Date(Math.min(...starts)).toISOString()
+    : null;
+  const readyValues = selected
+    .map((job) => secondsBetween(startedAt, job.completed_at))
+    .filter((value) => value !== null);
+  return {
+    expectedJobs: PREPARATION_JOBS.length,
+    observedJobs: selected.length,
+    reliable: selected.length === PREPARATION_JOBS.length
+      && selected.every((job) => job.conclusion === "success"),
+    startedAt,
+    readySeconds: readyValues.length === PREPARATION_JOBS.length
+      ? Math.max(...readyValues)
+      : null,
+    jobs: selected.map((job) => ({
+      name: job.name,
+      conclusion: job.conclusion ?? "unknown",
+      executionSeconds: secondsBetween(job.started_at, job.completed_at),
+      preparationRelativeSeconds: secondsBetween(startedAt, job.completed_at),
+    })),
+  };
+}
+
+function isMeaningfullyFaster(candidateSeconds, incumbentSeconds) {
+  if (candidateSeconds === null || incumbentSeconds === null) return false;
+  const absoluteGain = incumbentSeconds - candidateSeconds;
+  const relativeGain = absoluteGain / incumbentSeconds;
+  return absoluteGain >= MINIMUM_ABSOLUTE_GAIN_SECONDS
+    && relativeGain >= MINIMUM_RELATIVE_GAIN;
+}
+
+function selectWorkers(experiments) {
+  const candidates = experiments.filter((item) => item.imageMode === "prebuilt" && item.reliable);
+  let selected = candidates.find((item) => item.workers === 2) ?? null;
+  if (!selected) return null;
+  for (const candidate of candidates.filter((item) => item.workers > 2)) {
+    if (isMeaningfullyFaster(candidate.testCriticalPathSeconds, selected.testCriticalPathSeconds)) {
+      selected = candidate;
+    }
+  }
+  return selected;
+}
+
+export function compareRuntimeExperiments({ benchmarkRun, benchmarkJobs, totalTests }) {
+  if (!Number.isSafeInteger(totalTests) || totalTests < 1) {
+    throw new Error("totalTests must be a positive integer");
+  }
+  const jobs = benchmarkJobs.jobs ?? benchmarkJobs;
+  const experiments = EXPERIMENTS.map((experiment) =>
+    summarizeExperiment(experiment, benchmarkRun, jobs, totalTests));
+  const preparation = summarizePreparation(jobs);
+  const local = experiments.find((item) => item.id === "local-w2");
+  const prebuilt = experiments.find((item) => item.id === "prebuilt-w2");
+  const localEndToEndSeconds = local?.topologyReadySeconds ?? null;
+  const prebuiltEndToEndSeconds = preparation.readySeconds !== null
+    && prebuilt?.topologyReadySeconds !== null
+    ? preparation.readySeconds + prebuilt.topologyReadySeconds
+    : null;
+  const prebuiltImages = Boolean(
+    preparation.reliable
+      && local?.reliable
+      && prebuilt?.reliable
+      && isMeaningfullyFaster(prebuiltEndToEndSeconds, localEndToEndSeconds),
+  );
+  const selectedWorkers = prebuiltImages ? selectWorkers(experiments) : null;
+
+  return {
+    schemaVersion: 1,
+    headSha: benchmarkRun.head_sha,
+    policy: {
+      shards: SHARDS,
+      retries: 0,
+      candidates: EXPERIMENTS,
+      maximumChromiumShards: 20,
+      minimumRelativeGain: MINIMUM_RELATIVE_GAIN,
+      minimumAbsoluteGainSeconds: MINIMUM_ABSOLUTE_GAIN_SECONDS,
+      cacheSelectionMetric: "reliable parallel image preparation plus topology completion versus local topology completion at identical 16x2 test parallelism",
+      workerSelectionMetric: "reliable retry-free Playwright test critical path after image reuse",
+      status: "same commit; sequential experiments; parallel image preparation is charged to the reuse candidate; image reuse and worker count are measured independently",
+    },
+    preparation,
+    experiments,
+    cacheComparison: {
+      localEndToEndSeconds,
+      prebuiltEndToEndSeconds,
+    },
+    provisionalSelection: {
+      prebuiltImages,
+      workersPerShard: selectedWorkers?.workers ?? null,
+    },
+  };
+}
+
+function argument(name) {
+  const index = process.argv.indexOf(name);
+  return index === -1 ? undefined : process.argv[index + 1];
+}
+
+function main() {
+  for (const option of ["--benchmark-run", "--benchmark-jobs", "--total-tests", "--output"]) {
+    if (!argument(option)) throw new Error(`Missing ${option}`);
+  }
+  const result = compareRuntimeExperiments({
+    benchmarkRun: JSON.parse(fs.readFileSync(argument("--benchmark-run"), "utf8")),
+    benchmarkJobs: JSON.parse(fs.readFileSync(argument("--benchmark-jobs"), "utf8")),
+    totalTests: Number(argument("--total-tests")),
+  });
+  const output = argument("--output");
+  fs.mkdirSync(path.dirname(output), { recursive: true });
+  fs.writeFileSync(output, `${JSON.stringify(result, null, 2)}\n`);
+  console.log(
+    `Runtime benchmark selection: prebuilt=${result.provisionalSelection.prebuiltImages}; `
+      + `workers=${result.provisionalSelection.workersPerShard ?? "none"}`,
+  );
+}
+
+if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) main();