From 76fc7f2e41ac6e411c8cea13cce57cd0d2c981fb Mon Sep 17 00:00:00 2001 From: msrivas-7 Date: Mon, 31 Aug 2026 05:10:21 -0700 Subject: [PATCH 1/5] ci(e2e): benchmark build reuse and worker count --- .github/scripts/e2e-runtime-benchmark.mjs | 175 ++++++++++++++++ .../scripts/e2e-runtime-benchmark.test.mjs | 96 +++++++++ .github/workflows/ci.yml | 2 +- .github/workflows/e2e-runtime-benchmark.yml | 192 ++++++++++++++++++ .github/workflows/e2e-shard-topology.yml | 53 ++++- docker-compose.yml | 4 + e2e/README.md | 14 +- 7 files changed, 529 insertions(+), 7 deletions(-) create mode 100644 .github/scripts/e2e-runtime-benchmark.mjs create mode 100644 .github/scripts/e2e-runtime-benchmark.test.mjs create mode 100644 .github/workflows/e2e-runtime-benchmark.yml diff --git a/.github/scripts/e2e-runtime-benchmark.mjs b/.github/scripts/e2e-runtime-benchmark.mjs new file mode 100644 index 00000000..f320b255 --- /dev/null +++ b/.github/scripts/e2e-runtime-benchmark.mjs @@ -0,0 +1,175 @@ +#!/usr/bin/env node + +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const SHARDS = 16; +const MINIMUM_RELATIVE_GAIN = 0.05; +const MINIMUM_ABSOLUTE_GAIN_SECONDS = 20; +const BOOT_STEP_NAME = "Boot docker-compose stack"; +const TEST_STEP_NAME = "Run identical full suite without retries"; +const EXPERIMENTS = [ + { id: "local-w2", imageMode: "local-build", workers: 2 }, + { id: "prebuilt-w2", imageMode: "prebuilt", workers: 2 }, + { id: "prebuilt-w3", imageMode: "prebuilt", workers: 3 }, + { id: "prebuilt-w4", imageMode: "prebuilt", workers: 4 }, +]; + +function secondsBetween(start, end) { + if (!start || !end) return null; + const value = (Date.parse(end) - Date.parse(start)) / 1000; + return Number.isFinite(value) && value >= 0 ? value : null; +} + +function stepSeconds(job, stepName) { + const step = job.steps?.find((candidate) => candidate.name === stepName); + return step ? secondsBetween(step.started_at, step.completed_at) : null; +} + +function percentile(values, fraction) { + if (values.length === 0) return null; + const sorted = [...values].sort((left, right) => left - right); + return sorted[Math.ceil(fraction * sorted.length) - 1]; +} + +function summarizeExperiment(experiment, run, jobs, totalTests) { + const prefix = `Playwright runtime ${experiment.id} (`; + const selected = jobs.filter((job) => job.name.includes(prefix)); + const starts = selected.map((job) => Date.parse(job.started_at)).filter(Number.isFinite); + const topologyStartedAt = starts.length === SHARDS + ? new Date(Math.min(...starts)).toISOString() + : null; + const execution = selected + .map((job) => secondsBetween(job.started_at, job.completed_at)) + .filter((value) => value !== null); + const relativeCompletion = selected + .map((job) => secondsBetween(topologyStartedAt, job.completed_at)) + .filter((value) => value !== null); + const boots = selected.map((job) => stepSeconds(job, BOOT_STEP_NAME)).filter((value) => value !== null); + const tests = selected.map((job) => stepSeconds(job, TEST_STEP_NAME)).filter((value) => value !== null); + const reliable = selected.length === SHARDS + && selected.every((job) => job.conclusion === "success") + && boots.length === SHARDS + && tests.length === SHARDS; + + return { + ...experiment, + runId: run.id, + shards: SHARDS, + expectedJobs: SHARDS, + observedJobs: selected.length, + reliable, + totalTests, + topologyStartedAt, + topologyReadySeconds: relativeCompletion.length === SHARDS + ? Math.max(...relativeCompletion) + : null, + slowestExecutionSeconds: execution.length === SHARDS ? Math.max(...execution) : null, + aggregateExecutionSeconds: execution.length === SHARDS + ? execution.reduce((sum, value) => sum + value, 0) + : null, + bootSeconds: { + median: boots.length === SHARDS ? percentile(boots, 0.5) : null, + p90: boots.length === SHARDS ? percentile(boots, 0.9) : null, + max: boots.length === SHARDS ? Math.max(...boots) : null, + }, + testCriticalPathSeconds: tests.length === SHARDS ? Math.max(...tests) : null, + aggregateTestSeconds: tests.length === SHARDS + ? tests.reduce((sum, value) => sum + value, 0) + : null, + testImbalanceSeconds: tests.length === SHARDS + ? Math.max(...tests) - Math.min(...tests) + : null, + jobs: selected.map((job) => ({ + name: job.name, + conclusion: job.conclusion ?? "unknown", + executionSeconds: secondsBetween(job.started_at, job.completed_at), + bootSeconds: stepSeconds(job, BOOT_STEP_NAME), + testSeconds: stepSeconds(job, TEST_STEP_NAME), + })), + }; +} + +function isMeaningfullyFaster(candidateSeconds, incumbentSeconds) { + if (candidateSeconds === null || incumbentSeconds === null) return false; + const absoluteGain = incumbentSeconds - candidateSeconds; + const relativeGain = absoluteGain / incumbentSeconds; + return absoluteGain >= MINIMUM_ABSOLUTE_GAIN_SECONDS + && relativeGain >= MINIMUM_RELATIVE_GAIN; +} + +function selectWorkers(experiments) { + const candidates = experiments.filter((item) => item.imageMode === "prebuilt" && item.reliable); + let selected = candidates.find((item) => item.workers === 2) ?? null; + if (!selected) return null; + for (const candidate of candidates.filter((item) => item.workers > 2)) { + if (isMeaningfullyFaster(candidate.testCriticalPathSeconds, selected.testCriticalPathSeconds)) { + selected = candidate; + } + } + return selected; +} + +export function compareRuntimeExperiments({ benchmarkRun, benchmarkJobs, totalTests }) { + if (!Number.isSafeInteger(totalTests) || totalTests < 1) { + throw new Error("totalTests must be a positive integer"); + } + const jobs = benchmarkJobs.jobs ?? benchmarkJobs; + const experiments = EXPERIMENTS.map((experiment) => + summarizeExperiment(experiment, benchmarkRun, jobs, totalTests)); + const local = experiments.find((item) => item.id === "local-w2"); + const prebuilt = experiments.find((item) => item.id === "prebuilt-w2"); + const prebuiltImages = Boolean( + local?.reliable + && prebuilt?.reliable + && isMeaningfullyFaster(prebuilt.topologyReadySeconds, local.topologyReadySeconds), + ); + const selectedWorkers = prebuiltImages ? selectWorkers(experiments) : null; + + return { + schemaVersion: 1, + headSha: benchmarkRun.head_sha, + policy: { + shards: SHARDS, + retries: 0, + candidates: EXPERIMENTS, + maximumChromiumShards: 20, + minimumRelativeGain: MINIMUM_RELATIVE_GAIN, + minimumAbsoluteGainSeconds: MINIMUM_ABSOLUTE_GAIN_SECONDS, + cacheSelectionMetric: "reliable topology completion at identical 16x2 test parallelism", + workerSelectionMetric: "reliable retry-free Playwright test critical path after image reuse", + status: "same commit; sequential experiments; image reuse and worker count measured independently", + }, + experiments, + provisionalSelection: { + prebuiltImages, + workersPerShard: selectedWorkers?.workers ?? null, + }, + }; +} + +function argument(name) { + const index = process.argv.indexOf(name); + return index === -1 ? undefined : process.argv[index + 1]; +} + +function main() { + for (const option of ["--benchmark-run", "--benchmark-jobs", "--total-tests", "--output"]) { + if (!argument(option)) throw new Error(`Missing ${option}`); + } + const result = compareRuntimeExperiments({ + benchmarkRun: JSON.parse(fs.readFileSync(argument("--benchmark-run"), "utf8")), + benchmarkJobs: JSON.parse(fs.readFileSync(argument("--benchmark-jobs"), "utf8")), + totalTests: Number(argument("--total-tests")), + }); + const output = argument("--output"); + fs.mkdirSync(path.dirname(output), { recursive: true }); + fs.writeFileSync(output, `${JSON.stringify(result, null, 2)}\n`); + console.log( + `Runtime benchmark selection: prebuilt=${result.provisionalSelection.prebuiltImages}; ` + + `workers=${result.provisionalSelection.workersPerShard ?? "none"}`, + ); +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) main(); diff --git a/.github/scripts/e2e-runtime-benchmark.test.mjs b/.github/scripts/e2e-runtime-benchmark.test.mjs new file mode 100644 index 00000000..3b7e947c --- /dev/null +++ b/.github/scripts/e2e-runtime-benchmark.test.mjs @@ -0,0 +1,96 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import test from "node:test"; + +import { compareRuntimeExperiments } from "./e2e-runtime-benchmark.mjs"; + +const run = { id: 7, head_sha: "abc" }; +const workflow = readFileSync(new URL("../workflows/e2e-runtime-benchmark.yml", import.meta.url), "utf8"); +const topology = readFileSync(new URL("../workflows/e2e-shard-topology.yml", import.meta.url), "utf8"); +const compose = readFileSync(new URL("../../docker-compose.yml", import.meta.url), "utf8"); + +function job(experiment, shard, { duration, boot, tests, conclusion = "success", offset = 0 }) { + const start = new Date(Date.parse("2026-08-31T10:00:00Z") + offset * 1000); + const bootStart = new Date(start.getTime() + 10_000); + const bootEnd = new Date(bootStart.getTime() + boot * 1000); + const testStart = new Date(bootEnd.getTime() + 20_000); + const testEnd = new Date(testStart.getTime() + tests * 1000); + return { + name: `call / Playwright runtime ${experiment} (${shard}/16)`, + conclusion, + started_at: start.toISOString(), + completed_at: new Date(start.getTime() + duration * 1000).toISOString(), + steps: [ + { name: "Boot docker-compose stack", started_at: bootStart.toISOString(), completed_at: bootEnd.toISOString() }, + { name: "Run identical full suite without retries", started_at: testStart.toISOString(), completed_at: testEnd.toISOString() }, + ], + }; +} + +function experiment(id, values) { + return Array.from({ length: 16 }, (_, index) => job(id, index + 1, values)); +} + +function reliableJobs() { + return [ + ...experiment("local-w2", { duration: 360, boot: 120, tests: 190 }), + ...experiment("prebuilt-w2", { duration: 300, boot: 55, tests: 190, offset: 500 }), + ...experiment("prebuilt-w3", { duration: 270, boot: 55, tests: 160, offset: 900 }), + ...experiment("prebuilt-w4", { duration: 265, boot: 55, tests: 155, offset: 1_300 }), + ]; +} + +test("selects image reuse and only materially faster worker counts", () => { + const result = compareRuntimeExperiments({ + benchmarkRun: run, + benchmarkJobs: { jobs: reliableJobs() }, + totalTests: 439, + }); + assert.deepEqual(result.provisionalSelection, { prebuiltImages: true, workersPerShard: 3 }); + assert.equal(result.experiments[0].bootSeconds.median, 120); + assert.equal(result.experiments[1].bootSeconds.p90, 55); +}); + +test("does not adopt image reuse without a material end-to-end gain", () => { + const jobs = reliableJobs(); + for (const item of jobs.filter((candidate) => candidate.name.includes("prebuilt-w2"))) { + item.completed_at = new Date(Date.parse(item.started_at) + 350_000).toISOString(); + } + const result = compareRuntimeExperiments({ benchmarkRun: run, benchmarkJobs: jobs, totalTests: 439 }); + assert.deepEqual(result.provisionalSelection, { prebuiltImages: false, workersPerShard: null }); +}); + +test("never selects an unreliable worker experiment", () => { + const jobs = reliableJobs().filter((item) => !item.name.includes("prebuilt-w4")); + jobs.find((item) => item.name.includes("prebuilt-w3")).conclusion = "failure"; + const result = compareRuntimeExperiments({ benchmarkRun: run, benchmarkJobs: jobs, totalTests: 439 }); + assert.deepEqual(result.provisionalSelection, { prebuiltImages: true, workersPerShard: 2 }); + assert.equal(result.experiments.find((item) => item.id === "prebuilt-w3").reliable, false); +}); + +test("workflow holds shards constant and changes one worker variable per stage", () => { + assert.match(workflow, /experiment: local-w2\n\s+workers: 2/); + assert.match(workflow, /experiment: prebuilt-w2\n\s+workers: 2/); + assert.match(workflow, /experiment: prebuilt-w3\n\s+workers: 3/); + assert.match(workflow, /experiment: prebuilt-w4\n\s+workers: 4/); + assert.doesNotMatch(workflow, /max_parallel: (?:2[1-9]|[3-9][0-9])/); + assert.match(topology, /--workers=\$\{\{ inputs\.workers }}/); + assert.match(topology, /docker compose up -d --no-build backend frontend/); + assert.match(compose, /image: \$\{FRONTEND_IMAGE:-codetutor-ai-frontend:latest}/); +}); + +test("registry writes are restricted to labeled same-repository pull requests", () => { + assert.match(workflow, /packages: write/); + assert.match( + workflow, + /github\.event\.label\.name == 'ci-runtime-benchmark' && github\.event\.pull_request\.head\.repo\.full_name == github\.repository/, + ); + assert.doesNotMatch(workflow, /pull_request_target/); +}); + +test("rejects an invalid test inventory", () => { + assert.throws( + () => compareRuntimeExperiments({ benchmarkRun: run, benchmarkJobs: [], totalTests: 0 }), + /positive integer/, + ); +}); diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 09992eb1..dd03087a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ jobs: with: node-version: '20' - name: Test immutable manifest and VM promotion rollback - run: node --test .github/scripts/e2e-shadow-evidence.test.mjs .github/scripts/e2e-shard-benchmark.test.mjs .github/scripts/e2e-shard-capacity.test.mjs .github/scripts/frontend-release-probe.test.mjs .github/scripts/production-synthetic.test.mjs .github/scripts/release-manifest.test.mjs .github/scripts/release-permissions.test.mjs .github/scripts/vm-promote-candidate.test.mjs scripts/agent-harness.test.mjs scripts/e2e-shadow-contract.test.mjs scripts/production-dependency-audit.test.mjs + run: node --test .github/scripts/e2e-shadow-evidence.test.mjs .github/scripts/e2e-shard-benchmark.test.mjs .github/scripts/e2e-runtime-benchmark.test.mjs .github/scripts/e2e-shard-capacity.test.mjs .github/scripts/frontend-release-probe.test.mjs .github/scripts/production-synthetic.test.mjs .github/scripts/release-manifest.test.mjs .github/scripts/release-permissions.test.mjs .github/scripts/vm-promote-candidate.test.mjs scripts/agent-harness.test.mjs scripts/e2e-shadow-contract.test.mjs scripts/production-dependency-audit.test.mjs - name: Validate agent harness contract run: node scripts/agent-harness.mjs doctor --ci - name: Audit production dependencies diff --git a/.github/workflows/e2e-runtime-benchmark.yml b/.github/workflows/e2e-runtime-benchmark.yml new file mode 100644 index 00000000..b4900d00 --- /dev/null +++ b/.github/workflows/e2e-runtime-benchmark.yml @@ -0,0 +1,192 @@ +name: E2E runtime benchmark + +# Controlled, same-commit benchmark for the two remaining large E2E costs: +# rebuilding three Docker images in every shard and Playwright worker count. +# The selected 16-shard topology stays fixed throughout so each stage changes +# exactly one variable. Stages run sequentially and never exceed 16 concurrent +# shards, leaving headroom beneath the account-wide 40-job ceiling. +on: + pull_request: + types: [labeled] + +concurrency: + group: e2e-runtime-benchmark-${{ github.event.pull_request.number }} + cancel-in-progress: true + +permissions: + actions: read + contents: read + packages: write + pull-requests: read + +env: + CI_BACKEND_IMAGE: ghcr.io/${{ github.repository_owner }}/codetutor-ci-backend + CI_RUNNER_IMAGE: ghcr.io/${{ github.repository_owner }}/codetutor-ci-runner + CI_FRONTEND_IMAGE: ghcr.io/${{ github.repository_owner }}/codetutor-ci-frontend + +jobs: + prepare-images: + name: Build immutable E2E images once + if: github.event.label.name == 'ci-runtime-benchmark' && github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + outputs: + backend_ref: ${{ steps.refs.outputs.backend_ref }} + runner_ref: ${{ steps.refs.outputs.runner_ref }} + frontend_ref: ${{ steps.refs.outputs.frontend_ref }} + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + - name: Log in to GHCR + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + - name: Build and publish backend benchmark image + id: backend + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + with: + context: . + file: ./backend/Dockerfile + push: true + tags: ${{ env.CI_BACKEND_IMAGE }}:run-${{ github.run_id }} + cache-from: type=gha,scope=e2e-backend + cache-to: type=gha,scope=e2e-backend,mode=max + - name: Build and publish runner benchmark image + id: runner + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + with: + context: ./runner-image + push: true + tags: ${{ env.CI_RUNNER_IMAGE }}:run-${{ github.run_id }} + cache-from: type=gha,scope=e2e-runner + cache-to: type=gha,scope=e2e-runner,mode=max + - name: Build and publish frontend benchmark image + id: frontend + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + with: + context: . + file: ./frontend/Dockerfile + target: dev + push: true + tags: ${{ env.CI_FRONTEND_IMAGE }}:run-${{ github.run_id }} + cache-from: type=gha,scope=e2e-frontend + cache-to: type=gha,scope=e2e-frontend,mode=max + - name: Export immutable image references + id: refs + run: | + set -euo pipefail + { + echo "backend_ref=$CI_BACKEND_IMAGE@${{ steps.backend.outputs.digest }}" + echo "runner_ref=$CI_RUNNER_IMAGE@${{ steps.runner.outputs.digest }}" + echo "frontend_ref=$CI_FRONTEND_IMAGE@${{ steps.frontend.outputs.digest }}" + } >> "$GITHUB_OUTPUT" + + baseline-local-w2: + if: github.event.label.name == 'ci-runtime-benchmark' + needs: prepare-images + uses: ./.github/workflows/e2e-shard-topology.yml + with: + total: 16 + shards: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]' + max_parallel: 16 + experiment: local-w2 + workers: 2 + secrets: inherit + + prebuilt-w2: + if: always() && github.event.label.name == 'ci-runtime-benchmark' && needs.prepare-images.result == 'success' && needs.baseline-local-w2.result == 'success' + needs: [prepare-images, baseline-local-w2] + uses: ./.github/workflows/e2e-shard-topology.yml + with: + total: 16 + shards: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]' + max_parallel: 16 + experiment: prebuilt-w2 + workers: 2 + backend_image_ref: ${{ needs.prepare-images.outputs.backend_ref }} + runner_image_ref: ${{ needs.prepare-images.outputs.runner_ref }} + frontend_image_ref: ${{ needs.prepare-images.outputs.frontend_ref }} + secrets: inherit + + prebuilt-w3: + if: always() && github.event.label.name == 'ci-runtime-benchmark' && needs.prepare-images.result == 'success' && needs.prebuilt-w2.result == 'success' + needs: [prepare-images, prebuilt-w2] + uses: ./.github/workflows/e2e-shard-topology.yml + with: + total: 16 + shards: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]' + max_parallel: 16 + experiment: prebuilt-w3 + workers: 3 + backend_image_ref: ${{ needs.prepare-images.outputs.backend_ref }} + runner_image_ref: ${{ needs.prepare-images.outputs.runner_ref }} + frontend_image_ref: ${{ needs.prepare-images.outputs.frontend_ref }} + secrets: inherit + + prebuilt-w4: + if: always() && github.event.label.name == 'ci-runtime-benchmark' && needs.prepare-images.result == 'success' && needs.prebuilt-w3.result == 'success' + needs: [prepare-images, prebuilt-w3] + uses: ./.github/workflows/e2e-shard-topology.yml + with: + total: 16 + shards: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]' + max_parallel: 16 + experiment: prebuilt-w4 + workers: 4 + backend_image_ref: ${{ needs.prepare-images.outputs.backend_ref }} + runner_image_ref: ${{ needs.prepare-images.outputs.runner_ref }} + frontend_image_ref: ${{ needs.prepare-images.outputs.frontend_ref }} + secrets: inherit + + compare: + name: Compare Docker reuse and worker counts + if: always() && github.event.label.name == 'ci-runtime-benchmark' + needs: [prepare-images, baseline-local-w2, prebuilt-w2, prebuilt-w3, prebuilt-w4] + runs-on: ubuntu-latest + env: + SUPABASE_URL: ${{ secrets.SUPABASE_URL }} + SUPABASE_ANON_KEY: ${{ secrets.SUPABASE_ANON_KEY }} + SUPABASE_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_SERVICE_ROLE_KEY }} + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - name: Collect same-commit benchmark data + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + mkdir -p e2e/runtime-benchmark + gh api "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" > e2e/runtime-benchmark/benchmark-run.json + gh api --paginate "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/jobs?per_page=100" --slurp \ + | node -e 'let d=""; process.stdin.on("data",c=>d+=c).on("end",()=>{const pages=JSON.parse(d); process.stdout.write(JSON.stringify({jobs:pages.flatMap(page=>page.jobs||[])},null,2)+"\n")})' \ + > e2e/runtime-benchmark/benchmark-jobs.json + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: '20' + cache: npm + cache-dependency-path: e2e/package-lock.json + - name: Count the benchmarked Chromium tests + working-directory: e2e + run: | + set -euo pipefail + npm ci + total_tests=$(npx playwright test --list --project=chromium | sed -nE 's/^Total: ([0-9]+) tests.*/\1/p' | tail -1) + test -n "$total_tests" + echo "TOTAL_TESTS=$total_tests" >> "$GITHUB_ENV" + - name: Compare reliable experiments + run: | + set -euo pipefail + node .github/scripts/e2e-runtime-benchmark.mjs \ + --benchmark-run e2e/runtime-benchmark/benchmark-run.json \ + --benchmark-jobs e2e/runtime-benchmark/benchmark-jobs.json \ + --total-tests "$TOTAL_TESTS" \ + --output e2e/runtime-benchmark/result.json + cat e2e/runtime-benchmark/result.json >> "$GITHUB_STEP_SUMMARY" + - name: Upload benchmark result + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: e2e-runtime-benchmark-result + path: e2e/runtime-benchmark + retention-days: 90 + if-no-files-found: error diff --git a/.github/workflows/e2e-shard-topology.yml b/.github/workflows/e2e-shard-topology.yml index 03211ff5..5513243a 100644 --- a/.github/workflows/e2e-shard-topology.yml +++ b/.github/workflows/e2e-shard-topology.yml @@ -18,6 +18,31 @@ on: description: Maximum shard jobs allowed to execute simultaneously required: true type: number + experiment: + description: Stable experiment identifier used in job names and evidence + required: false + default: '' + type: string + workers: + description: Playwright workers per shard + required: false + default: 2 + type: number + backend_image_ref: + description: Optional immutable backend image reference + required: false + default: '' + type: string + runner_image_ref: + description: Optional immutable runner image reference + required: false + default: '' + type: string + frontend_image_ref: + description: Optional immutable frontend image reference + required: false + default: '' + type: string secrets: SUPABASE_URL: required: true @@ -36,10 +61,11 @@ on: permissions: contents: read + packages: read jobs: benchmark: - name: Playwright benchmark ${{ inputs.total }} shards (${{ matrix.shard }}/${{ inputs.total }}) + name: ${{ inputs.experiment != '' && format('Playwright runtime {0} ({1}/{2})', inputs.experiment, matrix.shard, inputs.total) || format('Playwright benchmark {0} shards ({1}/{0})', inputs.total, matrix.shard) }} runs-on: ubuntu-latest timeout-minutes: 25 strategy: @@ -60,15 +86,36 @@ jobs: MAX_SESSIONS_PER_USER: "60" MAX_SESSIONS_GLOBAL: "240" DOCKER_EXEC_CONCURRENCY: "16" + BACKEND_IMAGE: ${{ inputs.backend_image_ref }} + RUNNER_IMAGE: ${{ inputs.runner_image_ref }} + FRONTEND_IMAGE: ${{ inputs.frontend_image_ref }} steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Log in to GHCR for immutable benchmark images + if: inputs.backend_image_ref != '' && inputs.runner_image_ref != '' && inputs.frontend_image_ref != '' + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: node-version: '20' cache: npm cache-dependency-path: e2e/package-lock.json - name: Boot docker-compose stack - run: docker compose up -d backend frontend + run: | + set -euo pipefail + if [ -n "$BACKEND_IMAGE" ] && [ -n "$RUNNER_IMAGE" ] && [ -n "$FRONTEND_IMAGE" ]; then + docker pull "$BACKEND_IMAGE" + docker pull "$RUNNER_IMAGE" + docker pull "$FRONTEND_IMAGE" + docker compose up -d --no-build backend frontend + else + docker compose up -d backend frontend + fi - name: Install e2e deps working-directory: e2e run: npm ci @@ -84,7 +131,7 @@ jobs: run: npx playwright install --with-deps chromium - name: Run identical full suite without retries working-directory: e2e - run: npx playwright test --shard=${{ matrix.shard }}/${{ inputs.total }} --retries=0 + run: npx playwright test --shard=${{ matrix.shard }}/${{ inputs.total }} --workers=${{ inputs.workers }} --retries=0 - name: Dump docker-compose logs on failure if: failure() run: docker compose logs --no-color --tail=300 diff --git a/docker-compose.yml b/docker-compose.yml index 04bd6124..7101f63e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -181,6 +181,10 @@ services: max-file: "3" frontend: + # CI can provide a digest-pinned development image so every browser shard + # exercises the exact same frontend build without rebuilding it locally. + # Local development keeps the familiar Compose-built image below. + image: ${FRONTEND_IMAGE:-codetutor-ai-frontend:latest} build: context: . dockerfile: frontend/Dockerfile diff --git a/e2e/README.md b/e2e/README.md index 4602c3bc..13df820f 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -119,9 +119,8 @@ See `.github/workflows/e2e.yml`. The current PR model is: browser boundary retained for each. The earlier shard benchmark measured four, six, and eight shards on commit `c6aa5f0`; at the then-smaller suite size, six was fastest at 316 seconds versus 340 for eight and 495 for four. The suite has -since grown to 420 Chromium tests, so -`.github/workflows/e2e-shard-benchmark.yml` later compared 16 and 20 shards -sequentially on the same stable GitHub Pro commit and without retries. Run +since grown to 439 Chromium tests, so the capacity benchmark compared 16 and 20 +shards sequentially on the same stable GitHub Pro commit and without retries. Run [`33385421742`](https://github.com/msrivas-7/CodeTutor-AI/actions/runs/33385421742) selected sixteen shards: its retry-free test critical path was 160 seconds and its topology completed in 379 seconds, versus 198 and 416 seconds for 20 @@ -145,6 +144,15 @@ observed account concurrency changes. The matrix supports up to 256 jobs, but that syntax limit is not useful capacity unless the account can actually start the jobs concurrently. +The labeled `.github/workflows/e2e-runtime-benchmark.yml` runtime experiment +uses the `ci-runtime-benchmark` label and holds those sixteen shards constant. +It first compares the existing per-shard +Docker build with one digest-pinned backend, runner, and development-frontend +build reused by every shard, then measures two, three, and four Playwright +workers on the reused images. Each stage is sequential, retry-free, and must be +fully green. Image reuse is adopted only from a material end-to-end gain; +worker count is selected independently from the Playwright test critical path. + `.github/e2e-shard-capacity.json` records the measured decision. Shard 1 counts the live Chromium inventory and fails closed when it reaches 467 tests or falls to 411, one measured shard-workload from the 439-test baseline. Re-run the From 5d1b33ce21a9bbf5e1c81c83caf502a3c770f7f8 Mon Sep 17 00:00:00 2001 From: msrivas-7 Date: Mon, 31 Aug 2026 05:24:16 -0700 Subject: [PATCH 2/5] test(e2e): isolate anonymous quota identities --- .github/scripts/e2e-forwarded-ip.mjs | 29 ++++++++++++++++ .github/scripts/e2e-forwarded-ip.test.mjs | 41 +++++++++++++++++++++++ .github/workflows/ci.yml | 2 +- .github/workflows/e2e-shard-topology.yml | 4 ++- .github/workflows/e2e.yml | 9 +++++ .github/workflows/security.yml | 3 ++ docker-compose.yml | 3 ++ e2e/README.md | 4 +++ frontend/vite.config.ts | 9 +++++ 9 files changed, 102 insertions(+), 2 deletions(-) create mode 100644 .github/scripts/e2e-forwarded-ip.mjs create mode 100644 .github/scripts/e2e-forwarded-ip.test.mjs diff --git a/.github/scripts/e2e-forwarded-ip.mjs b/.github/scripts/e2e-forwarded-ip.mjs new file mode 100644 index 00000000..74cbc627 --- /dev/null +++ b/.github/scripts/e2e-forwarded-ip.mjs @@ -0,0 +1,29 @@ +#!/usr/bin/env node + +import { createHash } from "node:crypto"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +/** + * Map a CI job namespace onto an address in 2001:db8::/32, the IPv6 prefix + * reserved for documentation. Each isolated Docker stack then exercises the real + * trusted-proxy and per-IP quota path without sharing one database counter. + */ +export function e2eForwardedIp(namespace) { + const normalized = String(namespace ?? "").trim(); + if (!normalized || normalized.length > 240) { + throw new Error("E2E namespace must contain 1-240 characters"); + } + const digest = createHash("sha256").update(`codetutor-e2e-client-v1:${normalized}`).digest(); + const segments = Array.from({ length: 6 }, (_, index) => + digest.readUInt16BE(index * 2).toString(16)); + return `2001:db8:${segments.join(":")}`; +} + +function main() { + const namespace = process.argv[2]; + if (!namespace) throw new Error("Usage: e2e-forwarded-ip.mjs "); + console.log(e2eForwardedIp(namespace)); +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) main(); diff --git a/.github/scripts/e2e-forwarded-ip.test.mjs b/.github/scripts/e2e-forwarded-ip.test.mjs new file mode 100644 index 00000000..51f48db4 --- /dev/null +++ b/.github/scripts/e2e-forwarded-ip.test.mjs @@ -0,0 +1,41 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import test from "node:test"; + +import { e2eForwardedIp } from "./e2e-forwarded-ip.mjs"; + +const e2eWorkflow = readFileSync(new URL("../workflows/e2e.yml", import.meta.url), "utf8"); +const topologyWorkflow = readFileSync(new URL("../workflows/e2e-shard-topology.yml", import.meta.url), "utf8"); +const securityWorkflow = readFileSync(new URL("../workflows/security.yml", import.meta.url), "utf8"); +const compose = readFileSync(new URL("../../docker-compose.yml", import.meta.url), "utf8"); +const viteConfig = readFileSync(new URL("../../frontend/vite.config.ts", import.meta.url), "utf8"); + +test("derives a stable address from the reserved benchmark range", () => { + const value = e2eForwardedIp("shard-5-run33390478931-attempt1"); + assert.equal(value, e2eForwardedIp("shard-5-run33390478931-attempt1")); + assert.match(value, /^2001:db8(?::[0-9a-f]{1,4}){6}$/); +}); + +test("isolates shards, attempts, lanes, and benchmark stages", () => { + const namespaces = [ + "shard-5-run33390478931-attempt1", + "shard-6-run33390478931-attempt1", + "shard-5-run33390478931-attempt2", + "cross-browser-webkit-run33390478931-attempt1", + "benchmark-prebuilt-w3-5-run33390478931-attempt1", + ]; + assert.equal(new Set(namespaces.map(e2eForwardedIp)).size, namespaces.length); +}); + +test("rejects absent and unbounded namespaces", () => { + assert.throws(() => e2eForwardedIp(""), /1-240/); + assert.throws(() => e2eForwardedIp("x".repeat(241)), /1-240/); +}); + +test("every Compose-backed browser lane installs its isolated proxy identity", () => { + assert.equal((e2eWorkflow.match(/name: Allocate isolated anonymous client identity/g) ?? []).length, 3); + assert.equal((topologyWorkflow.match(/name: Allocate isolated anonymous client identity/g) ?? []).length, 1); + assert.equal((securityWorkflow.match(/name: Allocate isolated anonymous client identity/g) ?? []).length, 1); + assert.match(compose, /E2E_FORWARDED_FOR: "\$\{E2E_FORWARDED_FOR:-}"/); + assert.match(viteConfig, /"x-forwarded-for": e2eForwardedFor/); +}); diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dd03087a..7205b73f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ jobs: with: node-version: '20' - name: Test immutable manifest and VM promotion rollback - run: node --test .github/scripts/e2e-shadow-evidence.test.mjs .github/scripts/e2e-shard-benchmark.test.mjs .github/scripts/e2e-runtime-benchmark.test.mjs .github/scripts/e2e-shard-capacity.test.mjs .github/scripts/frontend-release-probe.test.mjs .github/scripts/production-synthetic.test.mjs .github/scripts/release-manifest.test.mjs .github/scripts/release-permissions.test.mjs .github/scripts/vm-promote-candidate.test.mjs scripts/agent-harness.test.mjs scripts/e2e-shadow-contract.test.mjs scripts/production-dependency-audit.test.mjs + run: node --test .github/scripts/e2e-forwarded-ip.test.mjs .github/scripts/e2e-shadow-evidence.test.mjs .github/scripts/e2e-shard-benchmark.test.mjs .github/scripts/e2e-runtime-benchmark.test.mjs .github/scripts/e2e-shard-capacity.test.mjs .github/scripts/frontend-release-probe.test.mjs .github/scripts/production-synthetic.test.mjs .github/scripts/release-manifest.test.mjs .github/scripts/release-permissions.test.mjs .github/scripts/vm-promote-candidate.test.mjs scripts/agent-harness.test.mjs scripts/e2e-shadow-contract.test.mjs scripts/production-dependency-audit.test.mjs - name: Validate agent harness contract run: node scripts/agent-harness.mjs doctor --ci - name: Audit production dependencies diff --git a/.github/workflows/e2e-shard-topology.yml b/.github/workflows/e2e-shard-topology.yml index 5513243a..a6585226 100644 --- a/.github/workflows/e2e-shard-topology.yml +++ b/.github/workflows/e2e-shard-topology.yml @@ -82,7 +82,7 @@ jobs: DATABASE_URL: ${{ secrets.DATABASE_URL }} BYOK_ENCRYPTION_KEY: ${{ secrets.BYOK_ENCRYPTION_KEY }} METRICS_TOKEN: e2e-test-metrics-token - E2E_USER_SUFFIX: benchmark-${{ inputs.total }}-${{ matrix.shard }}-run${{ github.run_id }}-attempt${{ github.run_attempt }} + E2E_USER_SUFFIX: benchmark-${{ inputs.experiment || inputs.total }}-${{ matrix.shard }}-run${{ github.run_id }}-attempt${{ github.run_attempt }} MAX_SESSIONS_PER_USER: "60" MAX_SESSIONS_GLOBAL: "240" DOCKER_EXEC_CONCURRENCY: "16" @@ -105,6 +105,8 @@ jobs: node-version: '20' cache: npm cache-dependency-path: e2e/package-lock.json + - name: Allocate isolated anonymous client identity + run: echo "E2E_FORWARDED_FOR=$(node .github/scripts/e2e-forwarded-ip.mjs "$E2E_USER_SUFFIX")" >> "$GITHUB_ENV" - name: Boot docker-compose stack run: | set -euo pipefail diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 4fdcdae3..8ae88226 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -84,6 +84,9 @@ jobs: cache: npm cache-dependency-path: e2e/package-lock.json + - name: Allocate isolated anonymous client identity + run: echo "E2E_FORWARDED_FOR=$(node .github/scripts/e2e-forwarded-ip.mjs "$E2E_USER_SUFFIX")" >> "$GITHUB_ENV" + - name: Boot docker-compose stack run: | if [ -n "$BACKEND_IMAGE" ] && [ -n "$RUNNER_IMAGE" ]; then @@ -239,6 +242,9 @@ jobs: cache: npm cache-dependency-path: e2e/package-lock.json + - name: Allocate isolated anonymous client identity + run: echo "E2E_FORWARDED_FOR=$(node .github/scripts/e2e-forwarded-ip.mjs "$E2E_USER_SUFFIX")" >> "$GITHUB_ENV" + - name: Boot docker-compose stack run: | if [ -n "$BACKEND_IMAGE" ] && [ -n "$RUNNER_IMAGE" ]; then @@ -356,6 +362,9 @@ jobs: cache: npm cache-dependency-path: e2e/package-lock.json + - name: Allocate isolated anonymous client identity + run: echo "E2E_FORWARDED_FOR=$(node .github/scripts/e2e-forwarded-ip.mjs "$E2E_USER_SUFFIX")" >> "$GITHUB_ENV" + - name: Boot docker-compose stack run: | if [ -n "$BACKEND_IMAGE" ] && [ -n "$RUNNER_IMAGE" ]; then diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 9c2bbfd5..a29c3274 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -95,6 +95,9 @@ jobs: cache: npm cache-dependency-path: e2e/package-lock.json + - name: Allocate isolated anonymous client identity + run: echo "E2E_FORWARDED_FOR=$(node .github/scripts/e2e-forwarded-ip.mjs "$E2E_USER_SUFFIX")" >> "$GITHUB_ENV" + - name: Install tcpdump for the host sentinel # tcpdump is preinstalled on ubuntu-latest. This step is a # belt-and-suspenders no-op on the default runner; keeps the diff --git a/docker-compose.yml b/docker-compose.yml index 7101f63e..4c84e32b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -212,6 +212,9 @@ services: VITE_BACKEND_URL: "${VITE_BACKEND_URL:-http://backend:4000}" VITE_SUPABASE_URL: "${VITE_SUPABASE_URL}" VITE_SUPABASE_ANON_KEY: "${VITE_SUPABASE_ANON_KEY}" + # CI-only trusted-proxy identity. Empty in local development and + # production, so ordinary browser requests retain their natural IP. + E2E_FORWARDED_FOR: "${E2E_FORWARDED_FOR:-}" ports: # Default loopback-only. Prod deploys override FRONTEND_BIND. - "${FRONTEND_BIND:-127.0.0.1}:${FRONTEND_PORT:-5173}:5173" diff --git a/e2e/README.md b/e2e/README.md index 13df820f..ba5ff043 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -112,6 +112,10 @@ See `.github/workflows/e2e.yml`. The current PR model is: - one advisory, zero-retry Chromium critical lane (currently 41 tests in 15 files); - CI retries retain diagnostic traces, but `failOnFlakyTests` makes a flaky result fail its shard so a targeted rerun cannot erase the original signal; +- each lane, shard, attempt, and benchmark stage receives a stable synthetic + address from the reserved `2001:db8::/32` range through the Vite proxy, so + the real per-IP abuse controls are tested without unrelated jobs sharing one + daily database counter; - versioned shadow evidence that records queue-inclusive readiness and any miss where the critical lane passes but the full suite fails. `e2e/shadow/regression-corpus.json` freezes the initial P0/P1 catch corpus. diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts index 604aced1..0ea76f8f 100644 --- a/frontend/vite.config.ts +++ b/frontend/vite.config.ts @@ -3,6 +3,8 @@ import react from "@vitejs/plugin-react"; import { courseRegistryPlugin } from "./scripts/vitePluginCourseRegistry"; import { discoveryBuildPlugin, discoverySitePlugin } from "./scripts/vitePluginDiscovery"; +const e2eForwardedFor = process.env.E2E_FORWARDED_FOR?.trim(); + export default defineConfig({ plugins: [ react(), @@ -25,6 +27,13 @@ export default defineConfig({ "/api": { target: process.env.VITE_BACKEND_URL ?? "http://localhost:4000", changeOrigin: true, + // GitHub-hosted shards all reach their isolated Compose stacks from + // the same Docker gateway address. An explicit CI-only identity keeps + // real per-IP quota coverage without making unrelated shards share a + // daily counter. Production and ordinary local Vite omit the env var. + ...(e2eForwardedFor + ? { headers: { "x-forwarded-for": e2eForwardedFor } } + : {}), }, }, }, From 1e4402aecdb0b2797ef60ca08209c9298b0aa04a Mon Sep 17 00:00:00 2001 From: msrivas-7 Date: Mon, 31 Aug 2026 05:33:04 -0700 Subject: [PATCH 3/5] fix(ci): charge image preparation in benchmark --- .github/scripts/e2e-runtime-benchmark.mjs | 53 ++++++++++- .../scripts/e2e-runtime-benchmark.test.mjs | 42 +++++++++ .github/workflows/e2e-runtime-benchmark.yml | 92 ++++++++++++------- 3 files changed, 152 insertions(+), 35 deletions(-) diff --git a/.github/scripts/e2e-runtime-benchmark.mjs b/.github/scripts/e2e-runtime-benchmark.mjs index f320b255..97a3ec3a 100644 --- a/.github/scripts/e2e-runtime-benchmark.mjs +++ b/.github/scripts/e2e-runtime-benchmark.mjs @@ -9,6 +9,11 @@ const MINIMUM_RELATIVE_GAIN = 0.05; const MINIMUM_ABSOLUTE_GAIN_SECONDS = 20; const BOOT_STEP_NAME = "Boot docker-compose stack"; const TEST_STEP_NAME = "Run identical full suite without retries"; +const PREPARATION_JOBS = [ + "Build backend E2E image once", + "Build runner E2E image once", + "Build frontend E2E image once", +]; const EXPERIMENTS = [ { id: "local-w2", imageMode: "local-build", workers: 2 }, { id: "prebuilt-w2", imageMode: "prebuilt", workers: 2 }, @@ -91,6 +96,34 @@ function summarizeExperiment(experiment, run, jobs, totalTests) { }; } +function summarizePreparation(jobs) { + const selected = jobs.filter((job) => + PREPARATION_JOBS.some((name) => job.name.includes(name))); + const starts = selected.map((job) => Date.parse(job.started_at)).filter(Number.isFinite); + const startedAt = starts.length === PREPARATION_JOBS.length + ? new Date(Math.min(...starts)).toISOString() + : null; + const readyValues = selected + .map((job) => secondsBetween(startedAt, job.completed_at)) + .filter((value) => value !== null); + return { + expectedJobs: PREPARATION_JOBS.length, + observedJobs: selected.length, + reliable: selected.length === PREPARATION_JOBS.length + && selected.every((job) => job.conclusion === "success"), + startedAt, + readySeconds: readyValues.length === PREPARATION_JOBS.length + ? Math.max(...readyValues) + : null, + jobs: selected.map((job) => ({ + name: job.name, + conclusion: job.conclusion ?? "unknown", + executionSeconds: secondsBetween(job.started_at, job.completed_at), + preparationRelativeSeconds: secondsBetween(startedAt, job.completed_at), + })), + }; +} + function isMeaningfullyFaster(candidateSeconds, incumbentSeconds) { if (candidateSeconds === null || incumbentSeconds === null) return false; const absoluteGain = incumbentSeconds - candidateSeconds; @@ -118,12 +151,19 @@ export function compareRuntimeExperiments({ benchmarkRun, benchmarkJobs, totalTe const jobs = benchmarkJobs.jobs ?? benchmarkJobs; const experiments = EXPERIMENTS.map((experiment) => summarizeExperiment(experiment, benchmarkRun, jobs, totalTests)); + const preparation = summarizePreparation(jobs); const local = experiments.find((item) => item.id === "local-w2"); const prebuilt = experiments.find((item) => item.id === "prebuilt-w2"); + const localEndToEndSeconds = local?.topologyReadySeconds ?? null; + const prebuiltEndToEndSeconds = preparation.readySeconds !== null + && prebuilt?.topologyReadySeconds !== null + ? preparation.readySeconds + prebuilt.topologyReadySeconds + : null; const prebuiltImages = Boolean( - local?.reliable + preparation.reliable + && local?.reliable && prebuilt?.reliable - && isMeaningfullyFaster(prebuilt.topologyReadySeconds, local.topologyReadySeconds), + && isMeaningfullyFaster(prebuiltEndToEndSeconds, localEndToEndSeconds), ); const selectedWorkers = prebuiltImages ? selectWorkers(experiments) : null; @@ -137,11 +177,16 @@ export function compareRuntimeExperiments({ benchmarkRun, benchmarkJobs, totalTe maximumChromiumShards: 20, minimumRelativeGain: MINIMUM_RELATIVE_GAIN, minimumAbsoluteGainSeconds: MINIMUM_ABSOLUTE_GAIN_SECONDS, - cacheSelectionMetric: "reliable topology completion at identical 16x2 test parallelism", + cacheSelectionMetric: "reliable parallel image preparation plus topology completion versus local topology completion at identical 16x2 test parallelism", workerSelectionMetric: "reliable retry-free Playwright test critical path after image reuse", - status: "same commit; sequential experiments; image reuse and worker count measured independently", + status: "same commit; sequential experiments; parallel image preparation is charged to the reuse candidate; image reuse and worker count are measured independently", }, + preparation, experiments, + cacheComparison: { + localEndToEndSeconds, + prebuiltEndToEndSeconds, + }, provisionalSelection: { prebuiltImages, workersPerShard: selectedWorkers?.workers ?? null, diff --git a/.github/scripts/e2e-runtime-benchmark.test.mjs b/.github/scripts/e2e-runtime-benchmark.test.mjs index 3b7e947c..354a5441 100644 --- a/.github/scripts/e2e-runtime-benchmark.test.mjs +++ b/.github/scripts/e2e-runtime-benchmark.test.mjs @@ -31,8 +31,22 @@ function experiment(id, values) { return Array.from({ length: 16 }, (_, index) => job(id, index + 1, values)); } +function preparation(name, duration = 25, conclusion = "success") { + const started = new Date("2026-08-31T09:55:00Z"); + return { + name, + conclusion, + started_at: started.toISOString(), + completed_at: new Date(started.getTime() + duration * 1000).toISOString(), + steps: [], + }; +} + function reliableJobs() { return [ + preparation("Build backend E2E image once"), + preparation("Build runner E2E image once", 20), + preparation("Build frontend E2E image once", 22), ...experiment("local-w2", { duration: 360, boot: 120, tests: 190 }), ...experiment("prebuilt-w2", { duration: 300, boot: 55, tests: 190, offset: 500 }), ...experiment("prebuilt-w3", { duration: 270, boot: 55, tests: 160, offset: 900 }), @@ -47,6 +61,11 @@ test("selects image reuse and only materially faster worker counts", () => { totalTests: 439, }); assert.deepEqual(result.provisionalSelection, { prebuiltImages: true, workersPerShard: 3 }); + assert.equal(result.preparation.readySeconds, 25); + assert.deepEqual(result.cacheComparison, { + localEndToEndSeconds: 360, + prebuiltEndToEndSeconds: 325, + }); assert.equal(result.experiments[0].bootSeconds.median, 120); assert.equal(result.experiments[1].bootSeconds.p90, 55); }); @@ -60,6 +79,27 @@ test("does not adopt image reuse without a material end-to-end gain", () => { assert.deepEqual(result.provisionalSelection, { prebuiltImages: false, workersPerShard: null }); }); +test("charges the complete image preparation critical path to reuse", () => { + const jobs = reliableJobs(); + for (const item of jobs.filter((candidate) => candidate.name.includes("E2E image once"))) { + item.completed_at = new Date(Date.parse(item.started_at) + 80_000).toISOString(); + } + const result = compareRuntimeExperiments({ benchmarkRun: run, benchmarkJobs: jobs, totalTests: 439 }); + assert.deepEqual(result.cacheComparison, { + localEndToEndSeconds: 360, + prebuiltEndToEndSeconds: 380, + }); + assert.deepEqual(result.provisionalSelection, { prebuiltImages: false, workersPerShard: null }); +}); + +test("fails closed when any required image preparation job fails", () => { + const jobs = reliableJobs(); + jobs.find((item) => item.name.includes("runner E2E image")).conclusion = "failure"; + const result = compareRuntimeExperiments({ benchmarkRun: run, benchmarkJobs: jobs, totalTests: 439 }); + assert.equal(result.preparation.reliable, false); + assert.deepEqual(result.provisionalSelection, { prebuiltImages: false, workersPerShard: null }); +}); + test("never selects an unreliable worker experiment", () => { const jobs = reliableJobs().filter((item) => !item.name.includes("prebuilt-w4")); jobs.find((item) => item.name.includes("prebuilt-w3")).conclusion = "failure"; @@ -73,6 +113,8 @@ test("workflow holds shards constant and changes one worker variable per stage", assert.match(workflow, /experiment: prebuilt-w2\n\s+workers: 2/); assert.match(workflow, /experiment: prebuilt-w3\n\s+workers: 3/); assert.match(workflow, /experiment: prebuilt-w4\n\s+workers: 4/); + assert.equal((workflow.match(/name: Build (?:backend|runner|frontend) E2E image once/g) ?? []).length, 3); + assert.match(workflow, /needs: \[prepare-backend, prepare-runner, prepare-frontend\]/); assert.doesNotMatch(workflow, /max_parallel: (?:2[1-9]|[3-9][0-9])/); assert.match(topology, /--workers=\$\{\{ inputs\.workers }}/); assert.match(topology, /docker compose up -d --no-build backend frontend/); diff --git a/.github/workflows/e2e-runtime-benchmark.yml b/.github/workflows/e2e-runtime-benchmark.yml index b4900d00..7740d2dd 100644 --- a/.github/workflows/e2e-runtime-benchmark.yml +++ b/.github/workflows/e2e-runtime-benchmark.yml @@ -25,14 +25,12 @@ env: CI_FRONTEND_IMAGE: ghcr.io/${{ github.repository_owner }}/codetutor-ci-frontend jobs: - prepare-images: - name: Build immutable E2E images once + prepare-backend: + name: Build backend E2E image once if: github.event.label.name == 'ci-runtime-benchmark' && github.event.pull_request.head.repo.full_name == github.repository runs-on: ubuntu-latest outputs: - backend_ref: ${{ steps.refs.outputs.backend_ref }} - runner_ref: ${{ steps.refs.outputs.runner_ref }} - frontend_ref: ${{ steps.refs.outputs.frontend_ref }} + ref: ${{ steps.ref.outputs.value }} steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 @@ -52,6 +50,25 @@ jobs: tags: ${{ env.CI_BACKEND_IMAGE }}:run-${{ github.run_id }} cache-from: type=gha,scope=e2e-backend cache-to: type=gha,scope=e2e-backend,mode=max + - name: Export immutable backend reference + id: ref + run: echo "value=$CI_BACKEND_IMAGE@${{ steps.backend.outputs.digest }}" >> "$GITHUB_OUTPUT" + + prepare-runner: + name: Build runner E2E image once + if: github.event.label.name == 'ci-runtime-benchmark' && github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + outputs: + ref: ${{ steps.ref.outputs.value }} + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + - name: Log in to GHCR + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} - name: Build and publish runner benchmark image id: runner uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 @@ -61,6 +78,25 @@ jobs: tags: ${{ env.CI_RUNNER_IMAGE }}:run-${{ github.run_id }} cache-from: type=gha,scope=e2e-runner cache-to: type=gha,scope=e2e-runner,mode=max + - name: Export immutable runner reference + id: ref + run: echo "value=$CI_RUNNER_IMAGE@${{ steps.runner.outputs.digest }}" >> "$GITHUB_OUTPUT" + + prepare-frontend: + name: Build frontend E2E image once + if: github.event.label.name == 'ci-runtime-benchmark' && github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + outputs: + ref: ${{ steps.ref.outputs.value }} + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + - name: Log in to GHCR + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} - name: Build and publish frontend benchmark image id: frontend uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 @@ -72,19 +108,13 @@ jobs: tags: ${{ env.CI_FRONTEND_IMAGE }}:run-${{ github.run_id }} cache-from: type=gha,scope=e2e-frontend cache-to: type=gha,scope=e2e-frontend,mode=max - - name: Export immutable image references - id: refs - run: | - set -euo pipefail - { - echo "backend_ref=$CI_BACKEND_IMAGE@${{ steps.backend.outputs.digest }}" - echo "runner_ref=$CI_RUNNER_IMAGE@${{ steps.runner.outputs.digest }}" - echo "frontend_ref=$CI_FRONTEND_IMAGE@${{ steps.frontend.outputs.digest }}" - } >> "$GITHUB_OUTPUT" + - name: Export immutable frontend reference + id: ref + run: echo "value=$CI_FRONTEND_IMAGE@${{ steps.frontend.outputs.digest }}" >> "$GITHUB_OUTPUT" baseline-local-w2: if: github.event.label.name == 'ci-runtime-benchmark' - needs: prepare-images + needs: [prepare-backend, prepare-runner, prepare-frontend] uses: ./.github/workflows/e2e-shard-topology.yml with: total: 16 @@ -95,8 +125,8 @@ jobs: secrets: inherit prebuilt-w2: - if: always() && github.event.label.name == 'ci-runtime-benchmark' && needs.prepare-images.result == 'success' && needs.baseline-local-w2.result == 'success' - needs: [prepare-images, baseline-local-w2] + if: always() && github.event.label.name == 'ci-runtime-benchmark' && needs.prepare-backend.result == 'success' && needs.prepare-runner.result == 'success' && needs.prepare-frontend.result == 'success' && needs.baseline-local-w2.result == 'success' + needs: [prepare-backend, prepare-runner, prepare-frontend, baseline-local-w2] uses: ./.github/workflows/e2e-shard-topology.yml with: total: 16 @@ -104,14 +134,14 @@ jobs: max_parallel: 16 experiment: prebuilt-w2 workers: 2 - backend_image_ref: ${{ needs.prepare-images.outputs.backend_ref }} - runner_image_ref: ${{ needs.prepare-images.outputs.runner_ref }} - frontend_image_ref: ${{ needs.prepare-images.outputs.frontend_ref }} + backend_image_ref: ${{ needs.prepare-backend.outputs.ref }} + runner_image_ref: ${{ needs.prepare-runner.outputs.ref }} + frontend_image_ref: ${{ needs.prepare-frontend.outputs.ref }} secrets: inherit prebuilt-w3: - if: always() && github.event.label.name == 'ci-runtime-benchmark' && needs.prepare-images.result == 'success' && needs.prebuilt-w2.result == 'success' - needs: [prepare-images, prebuilt-w2] + if: always() && github.event.label.name == 'ci-runtime-benchmark' && needs.prepare-backend.result == 'success' && needs.prepare-runner.result == 'success' && needs.prepare-frontend.result == 'success' && needs.prebuilt-w2.result == 'success' + needs: [prepare-backend, prepare-runner, prepare-frontend, prebuilt-w2] uses: ./.github/workflows/e2e-shard-topology.yml with: total: 16 @@ -119,14 +149,14 @@ jobs: max_parallel: 16 experiment: prebuilt-w3 workers: 3 - backend_image_ref: ${{ needs.prepare-images.outputs.backend_ref }} - runner_image_ref: ${{ needs.prepare-images.outputs.runner_ref }} - frontend_image_ref: ${{ needs.prepare-images.outputs.frontend_ref }} + backend_image_ref: ${{ needs.prepare-backend.outputs.ref }} + runner_image_ref: ${{ needs.prepare-runner.outputs.ref }} + frontend_image_ref: ${{ needs.prepare-frontend.outputs.ref }} secrets: inherit prebuilt-w4: - if: always() && github.event.label.name == 'ci-runtime-benchmark' && needs.prepare-images.result == 'success' && needs.prebuilt-w3.result == 'success' - needs: [prepare-images, prebuilt-w3] + if: always() && github.event.label.name == 'ci-runtime-benchmark' && needs.prepare-backend.result == 'success' && needs.prepare-runner.result == 'success' && needs.prepare-frontend.result == 'success' && needs.prebuilt-w3.result == 'success' + needs: [prepare-backend, prepare-runner, prepare-frontend, prebuilt-w3] uses: ./.github/workflows/e2e-shard-topology.yml with: total: 16 @@ -134,15 +164,15 @@ jobs: max_parallel: 16 experiment: prebuilt-w4 workers: 4 - backend_image_ref: ${{ needs.prepare-images.outputs.backend_ref }} - runner_image_ref: ${{ needs.prepare-images.outputs.runner_ref }} - frontend_image_ref: ${{ needs.prepare-images.outputs.frontend_ref }} + backend_image_ref: ${{ needs.prepare-backend.outputs.ref }} + runner_image_ref: ${{ needs.prepare-runner.outputs.ref }} + frontend_image_ref: ${{ needs.prepare-frontend.outputs.ref }} secrets: inherit compare: name: Compare Docker reuse and worker counts if: always() && github.event.label.name == 'ci-runtime-benchmark' - needs: [prepare-images, baseline-local-w2, prebuilt-w2, prebuilt-w3, prebuilt-w4] + needs: [prepare-backend, prepare-runner, prepare-frontend, baseline-local-w2, prebuilt-w2, prebuilt-w3, prebuilt-w4] runs-on: ubuntu-latest env: SUPABASE_URL: ${{ secrets.SUPABASE_URL }} From 60e20db6ae8686e33cb4331c993e79af8f5c7cbc Mon Sep 17 00:00:00 2001 From: msrivas-7 Date: Mon, 31 Aug 2026 06:19:04 -0700 Subject: [PATCH 4/5] test(e2e): tolerate teardown transport failures --- e2e/fixtures/auth.ts | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/e2e/fixtures/auth.ts b/e2e/fixtures/auth.ts index 792ea399..11a0e1d1 100644 --- a/e2e/fixtures/auth.ts +++ b/e2e/fixtures/auth.ts @@ -394,16 +394,29 @@ export function trackSessionCleanup( // resume endpoint. This also covers sessions created from peer tabs that // disappeared before their response event reached the coordinator. for (let attempt = 0; attempt < 20; attempt += 1) { - const resume = await ctx.post(`${BACKEND_URL}/api/session/resume`); + let resume: import("@playwright/test").APIResponse; + try { + resume = await ctx.post(`${BACKEND_URL}/api/session/resume`); + } catch { + // The product assertion has already completed. A backend shutdown or + // transient socket failure during this best-effort drain must not + // retroactively fail it; the idle-session sweeper is the final guard. + break; + } if (resume.status() === 404) break; if (!resume.ok()) break; const body = (await resume.json().catch(() => null)) as { sessionId?: unknown; } | null; if (typeof body?.sessionId !== "string" || body.sessionId.length === 0) break; - const ended = await ctx.post(`${BACKEND_URL}/api/session/end`, { - data: { sessionId: body.sessionId }, - }); + let ended: import("@playwright/test").APIResponse; + try { + ended = await ctx.post(`${BACKEND_URL}/api/session/end`, { + data: { sessionId: body.sessionId }, + }); + } catch { + break; + } if (!ended.ok()) break; } } finally { From f3ee51e3cc08ac53af549c01805c3fbfb12dae5c Mon Sep 17 00:00:00 2001 From: msrivas-7 Date: Mon, 31 Aug 2026 07:02:48 -0700 Subject: [PATCH 5/5] ci(e2e): reuse measured build artifacts --- .github/e2e-shard-capacity.json | 21 ++ .github/scripts/e2e-shard-capacity.test.mjs | 19 ++ .github/scripts/ghcr-retention.mjs | 150 +++++++++++++ .github/scripts/ghcr-retention.test.mjs | 78 +++++++ .github/workflows/e2e-image-retention.yml | 37 +++ .github/workflows/e2e.yml | 237 ++++++++++++++++++-- 6 files changed, 526 insertions(+), 16 deletions(-) create mode 100644 .github/scripts/ghcr-retention.mjs create mode 100644 .github/scripts/ghcr-retention.test.mjs create mode 100644 .github/workflows/e2e-image-retention.yml diff --git a/.github/e2e-shard-capacity.json b/.github/e2e-shard-capacity.json index 41eda573..a280a3b9 100644 --- a/.github/e2e-shard-capacity.json +++ b/.github/e2e-shard-capacity.json @@ -33,5 +33,26 @@ "minimumRelativeGain": 0.05, "requiresEveryShardToPass": true, "preservesFullChromiumSuite": true + }, + "runtimeOptimization": { + "runId": 33397529873, + "url": "https://github.com/msrivas-7/CodeTutor-AI/actions/runs/33397529873", + "headSha": "60e20db6ae8686e33cb4331c993e79af8f5c7cbc", + "totalTests": 439, + "imageReuse": { + "localBuildEndToEndSeconds": 369, + "prebuiltEndToEndSecondsIncludingPreparation": 338, + "preparationSeconds": 24, + "absoluteGainSeconds": 31, + "relativeGain": 0.084, + "selected": true + }, + "workerCandidates": [ + { "workers": 2, "testCriticalPathSeconds": 190, "reliable": true, "selected": true }, + { "workers": 3, "testCriticalPathSeconds": 160, "reliable": false, "selected": false }, + { "workers": 4, "testCriticalPathSeconds": null, "reliable": false, "selected": false } + ], + "maximumChromiumShards": 20, + "method": "Three images prepared in parallel, then local-build and digest-pinned 16-shard stages ran sequentially on the same commit with zero retries. Higher worker counts were eligible only when every shard passed." } } diff --git a/.github/scripts/e2e-shard-capacity.test.mjs b/.github/scripts/e2e-shard-capacity.test.mjs index 60d5aa56..8603d659 100644 --- a/.github/scripts/e2e-shard-capacity.test.mjs +++ b/.github/scripts/e2e-shard-capacity.test.mjs @@ -41,6 +41,25 @@ test("tracked decision preserves the clean controlled benchmark evidence", () => ], ); assert.equal(record.benchmark.selectedModeledTestCriticalPathSeconds, 160); + assert.deepEqual(record.runtimeOptimization.imageReuse, { + localBuildEndToEndSeconds: 369, + prebuiltEndToEndSecondsIncludingPreparation: 338, + preparationSeconds: 24, + absoluteGainSeconds: 31, + relativeGain: 0.084, + selected: true, + }); + assert.deepEqual( + record.runtimeOptimization.workerCandidates.map( + ({ workers, reliable, selected }) => ({ workers, reliable, selected }), + ), + [ + { workers: 2, reliable: true, selected: true }, + { workers: 3, reliable: false, selected: false }, + { workers: 4, reliable: false, selected: false }, + ], + ); + assert.equal(record.runtimeOptimization.maximumChromiumShards, 20); }); test("blocking workflow uses the selected matrix and derives its denominator", () => { diff --git a/.github/scripts/ghcr-retention.mjs b/.github/scripts/ghcr-retention.mjs new file mode 100644 index 00000000..6257364b --- /dev/null +++ b/.github/scripts/ghcr-retention.mjs @@ -0,0 +1,150 @@ +#!/usr/bin/env node + +import { pathToFileURL } from "node:url"; + +const API_ROOT = process.env.GITHUB_API_URL || "https://api.github.com"; + +export function selectPackageVersions( + versions, + { tag = null, olderThanMs = null, keepNewest = 0, now = Date.now() } = {}, +) { + const newestFirst = [...versions].sort( + (a, b) => Date.parse(b.updated_at) - Date.parse(a.updated_at), + ); + + if (tag) { + return newestFirst.filter((version) => + (version.metadata?.container?.tags ?? []).includes(tag), + ); + } + + if (!Number.isFinite(olderThanMs) || olderThanMs < 0) { + throw new Error("Provide either --tag or a non-negative --older-than-hours value"); + } + + const protectedIds = new Set( + newestFirst.slice(0, Math.max(0, keepNewest)).map((version) => version.id), + ); + const cutoff = now - olderThanMs; + return newestFirst.filter( + (version) => + !protectedIds.has(version.id) && Date.parse(version.updated_at) < cutoff, + ); +} + +function parseArgs(argv) { + const options = { requireMatch: false, keepNewest: 0 }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--require-match") { + options.requireMatch = true; + continue; + } + const value = argv[index + 1]; + if (!value || value.startsWith("--")) throw new Error(`Missing value for ${arg}`); + index += 1; + if (arg === "--owner") options.owner = value; + else if (arg === "--package") options.package = value; + else if (arg === "--tag") options.tag = value; + else if (arg === "--older-than-hours") options.olderThanHours = Number(value); + else if (arg === "--keep-newest") options.keepNewest = Number(value); + else throw new Error(`Unknown argument: ${arg}`); + } + if (!options.owner || !options.package) { + throw new Error("--owner and --package are required"); + } + if (options.tag && options.olderThanHours !== undefined) { + throw new Error("--tag and --older-than-hours are mutually exclusive"); + } + if (!options.tag && !Number.isFinite(options.olderThanHours)) { + throw new Error("Provide --tag or --older-than-hours"); + } + if (!Number.isInteger(options.keepNewest) || options.keepNewest < 0) { + throw new Error("--keep-newest must be a non-negative integer"); + } + return options; +} + +function nextLink(header) { + if (!header) return null; + const match = header + .split(",") + .map((part) => part.trim()) + .find((part) => /rel="next"/.test(part)); + return match?.match(/^<([^>]+)>/)?.[1] ?? null; +} + +async function githubRequest(url, token, init = {}) { + const response = await fetch(url.startsWith("http") ? url : `${API_ROOT}${url}`, { + ...init, + headers: { + Accept: "application/vnd.github+json", + Authorization: `Bearer ${token}`, + "X-GitHub-Api-Version": "2022-11-28", + ...init.headers, + }, + }); + if (!response.ok) { + const body = await response.text(); + throw new Error(`${init.method ?? "GET"} ${url} failed (${response.status}): ${body}`); + } + return response; +} + +async function listVersions(path, token) { + const versions = []; + let next = `${API_ROOT}${path}?per_page=100`; + while (next) { + const response = await githubRequest(next, token); + versions.push(...(await response.json())); + next = nextLink(response.headers.get("link")); + } + return versions; +} + +export async function run(argv = process.argv.slice(2)) { + const options = parseArgs(argv); + const token = process.env.GH_TOKEN || process.env.GITHUB_TOKEN; + if (!token) throw new Error("GH_TOKEN or GITHUB_TOKEN is required"); + + const ownerResponse = await githubRequest( + `/users/${encodeURIComponent(options.owner)}`, + token, + ); + const owner = await ownerResponse.json(); + const scope = owner.type === "Organization" ? "orgs" : "users"; + const packagePath = `/${scope}/${encodeURIComponent(options.owner)}/packages/container/${encodeURIComponent(options.package)}`; + const versions = await listVersions(`${packagePath}/versions`, token); + const selected = selectPackageVersions(versions, { + tag: options.tag, + olderThanMs: + options.olderThanHours === undefined + ? null + : options.olderThanHours * 60 * 60 * 1000, + keepNewest: options.keepNewest, + }); + + if (options.requireMatch && selected.length === 0) { + throw new Error(`No ${options.package} version matched tag ${options.tag}`); + } + + for (const version of selected) { + await githubRequest(`${packagePath}/versions/${version.id}`, token, { + method: "DELETE", + }); + } + console.log( + JSON.stringify({ + package: options.package, + deleted: selected.map((version) => version.id), + matched: selected.length, + }), + ); +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + run().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/.github/scripts/ghcr-retention.test.mjs b/.github/scripts/ghcr-retention.test.mjs new file mode 100644 index 00000000..752fc933 --- /dev/null +++ b/.github/scripts/ghcr-retention.test.mjs @@ -0,0 +1,78 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; + +import { selectPackageVersions } from "./ghcr-retention.mjs"; + +const versions = [ + { + id: 3, + updated_at: "2026-08-31T12:00:00Z", + metadata: { container: { tags: ["run-300"] } }, + }, + { + id: 2, + updated_at: "2026-08-28T12:00:00Z", + metadata: { container: { tags: ["run-200", "shared"] } }, + }, + { + id: 1, + updated_at: "2026-08-20T12:00:00Z", + metadata: { container: { tags: [] } }, + }, +]; + +test("exact-tag cleanup cannot select adjacent or untagged versions", () => { + assert.deepEqual( + selectPackageVersions(versions, { tag: "run-200" }).map((version) => version.id), + [2], + ); +}); + +test("retention removes only expired versions outside the newest safety window", () => { + assert.deepEqual( + selectPackageVersions(versions, { + olderThanMs: 48 * 60 * 60 * 1000, + keepNewest: 1, + now: Date.parse("2026-08-31T13:00:00Z"), + }).map((version) => version.id), + [2, 1], + ); +}); + +test("retention protects the newest requested versions even when all are old", () => { + assert.deepEqual( + selectPackageVersions(versions, { + olderThanMs: 0, + keepNewest: 2, + now: Date.parse("2026-09-01T00:00:00Z"), + }).map((version) => version.id), + [1], + ); +}); + +test("blocking E2E adopts digest reuse but cleans images only after retry-safe success", async () => { + const workflow = await readFile( + new URL("../workflows/e2e.yml", import.meta.url), + "utf8", + ); + assert.match(workflow, /name: Prepare backend E2E image/); + assert.match(workflow, /name: Prepare runner E2E image/); + assert.match(workflow, /name: Prepare frontend E2E image/); + assert.match(workflow, /FRONTEND_IMAGE: \$\{\{ needs\.prepare-frontend\.outputs\.ref \}\}/); + assert.match(workflow, /docker compose up -d --no-build backend frontend/); + assert.match(workflow, /needs\.e2e\.result == 'success'/); + assert.match(workflow, /needs\.cross-browser-core\.result == 'success'/); + assert.match(workflow, /--tag "\$RUN_TAG" --require-match/); +}); + +test("scheduled retention preserves one fallback and bounds stale versions", async () => { + const workflow = await readFile( + new URL("../workflows/e2e-image-retention.yml", import.meta.url), + "utf8", + ); + assert.match(workflow, /schedule:/); + assert.match(workflow, /--older-than-hours 48/); + assert.match(workflow, /--keep-newest 1/); + assert.match(workflow, /packages: write/); +}); diff --git a/.github/workflows/e2e-image-retention.yml b/.github/workflows/e2e-image-retention.yml new file mode 100644 index 00000000..d33a3374 --- /dev/null +++ b/.github/workflows/e2e-image-retention.yml @@ -0,0 +1,37 @@ +name: E2E image retention + +on: + schedule: + - cron: '23 9 * * *' + workflow_dispatch: + +permissions: + contents: read + packages: write + +concurrency: + group: e2e-image-retention + cancel-in-progress: false + +jobs: + cleanup: + name: Remove stale temporary E2E images + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + package: + - codetutor-ci-backend + - codetutor-ci-runner + - codetutor-ci-frontend + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - name: Keep one fallback and remove versions older than 48 hours + env: + GH_TOKEN: ${{ github.token }} + run: >- + node .github/scripts/ghcr-retention.mjs + --owner "$GITHUB_REPOSITORY_OWNER" + --package "${{ matrix.package }}" + --older-than-hours 48 + --keep-newest 1 diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 8ae88226..c786dc2b 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -30,6 +30,11 @@ on: required: false default: '' type: string + frontend_image_ref: + description: Immutable development-frontend digest to test; empty builds once + required: false + default: '' + type: string concurrency: group: e2e-${{ github.ref }} @@ -38,12 +43,165 @@ concurrency: permissions: actions: read contents: read - packages: read + packages: write pull-requests: read +env: + CI_BACKEND_IMAGE: ghcr.io/${{ github.repository_owner }}/codetutor-ci-backend + CI_RUNNER_IMAGE: ghcr.io/${{ github.repository_owner }}/codetutor-ci-runner + CI_FRONTEND_IMAGE: ghcr.io/${{ github.repository_owner }}/codetutor-ci-frontend + jobs: + prepare-backend: + name: Prepare backend E2E image + runs-on: ubuntu-latest + outputs: + ref: ${{ steps.ref.outputs.value }} + created: ${{ steps.ref.outputs.created }} + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - name: Configure Docker Buildx + if: inputs.backend_image_ref == '' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + - name: Log in to GHCR + if: inputs.backend_image_ref == '' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + - name: Build and publish backend E2E image + id: backend + if: inputs.backend_image_ref == '' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + with: + context: . + file: ./backend/Dockerfile + push: true + tags: ${{ env.CI_BACKEND_IMAGE }}:run-${{ github.run_id }} + cache-from: type=gha,scope=e2e-backend + cache-to: type=gha,scope=e2e-backend,mode=max + - name: Export backend reference + id: ref + env: + SUPPLIED_REF: ${{ inputs.backend_image_ref || '' }} + BUILT_DIGEST: ${{ steps.backend.outputs.digest }} + TRUSTED_HEAD: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} + run: | + set -euo pipefail + if [ -n "$SUPPLIED_REF" ]; then + echo "value=$SUPPLIED_REF" >> "$GITHUB_OUTPUT" + echo "created=false" >> "$GITHUB_OUTPUT" + elif [ "$TRUSTED_HEAD" = "true" ]; then + test -n "$BUILT_DIGEST" + echo "value=$CI_BACKEND_IMAGE@$BUILT_DIGEST" >> "$GITHUB_OUTPUT" + echo "created=true" >> "$GITHUB_OUTPUT" + else + echo "value=" >> "$GITHUB_OUTPUT" + echo "created=false" >> "$GITHUB_OUTPUT" + fi + + prepare-runner: + name: Prepare runner E2E image + runs-on: ubuntu-latest + outputs: + ref: ${{ steps.ref.outputs.value }} + created: ${{ steps.ref.outputs.created }} + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - name: Configure Docker Buildx + if: inputs.runner_image_ref == '' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + - name: Log in to GHCR + if: inputs.runner_image_ref == '' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + - name: Build and publish runner E2E image + id: runner + if: inputs.runner_image_ref == '' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + with: + context: ./runner-image + push: true + tags: ${{ env.CI_RUNNER_IMAGE }}:run-${{ github.run_id }} + cache-from: type=gha,scope=e2e-runner + cache-to: type=gha,scope=e2e-runner,mode=max + - name: Export runner reference + id: ref + env: + SUPPLIED_REF: ${{ inputs.runner_image_ref || '' }} + BUILT_DIGEST: ${{ steps.runner.outputs.digest }} + TRUSTED_HEAD: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} + run: | + set -euo pipefail + if [ -n "$SUPPLIED_REF" ]; then + echo "value=$SUPPLIED_REF" >> "$GITHUB_OUTPUT" + echo "created=false" >> "$GITHUB_OUTPUT" + elif [ "$TRUSTED_HEAD" = "true" ]; then + test -n "$BUILT_DIGEST" + echo "value=$CI_RUNNER_IMAGE@$BUILT_DIGEST" >> "$GITHUB_OUTPUT" + echo "created=true" >> "$GITHUB_OUTPUT" + else + echo "value=" >> "$GITHUB_OUTPUT" + echo "created=false" >> "$GITHUB_OUTPUT" + fi + + prepare-frontend: + name: Prepare frontend E2E image + runs-on: ubuntu-latest + outputs: + ref: ${{ steps.ref.outputs.value }} + created: ${{ steps.ref.outputs.created }} + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - name: Configure Docker Buildx + if: inputs.frontend_image_ref == '' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + - name: Log in to GHCR + if: inputs.frontend_image_ref == '' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + - name: Build and publish frontend E2E image + id: frontend + if: inputs.frontend_image_ref == '' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + with: + context: . + file: ./frontend/Dockerfile + target: dev + push: true + tags: ${{ env.CI_FRONTEND_IMAGE }}:run-${{ github.run_id }} + cache-from: type=gha,scope=e2e-frontend + cache-to: type=gha,scope=e2e-frontend,mode=max + - name: Export frontend reference + id: ref + env: + SUPPLIED_REF: ${{ inputs.frontend_image_ref || '' }} + BUILT_DIGEST: ${{ steps.frontend.outputs.digest }} + TRUSTED_HEAD: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} + run: | + set -euo pipefail + if [ -n "$SUPPLIED_REF" ]; then + echo "value=$SUPPLIED_REF" >> "$GITHUB_OUTPUT" + echo "created=false" >> "$GITHUB_OUTPUT" + elif [ "$TRUSTED_HEAD" = "true" ]; then + test -n "$BUILT_DIGEST" + echo "value=$CI_FRONTEND_IMAGE@$BUILT_DIGEST" >> "$GITHUB_OUTPUT" + echo "created=true" >> "$GITHUB_OUTPUT" + else + echo "value=" >> "$GITHUB_OUTPUT" + echo "created=false" >> "$GITHUB_OUTPUT" + fi + critical-shadow: name: Playwright critical lane (advisory) + needs: [prepare-backend, prepare-runner, prepare-frontend] runs-on: ubuntu-latest timeout-minutes: 20 # Release 1D begins in shadow mode. The exhaustive Chromium shards below @@ -65,13 +223,14 @@ jobs: MAX_SESSIONS_PER_USER: "60" MAX_SESSIONS_GLOBAL: "200" DOCKER_EXEC_CONCURRENCY: "16" - BACKEND_IMAGE: ${{ inputs.backend_image_ref || '' }} - RUNNER_IMAGE: ${{ inputs.runner_image_ref || '' }} + BACKEND_IMAGE: ${{ needs.prepare-backend.outputs.ref }} + RUNNER_IMAGE: ${{ needs.prepare-runner.outputs.ref }} + FRONTEND_IMAGE: ${{ needs.prepare-frontend.outputs.ref }} steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Log in to GHCR for immutable candidates - if: inputs.backend_image_ref != '' && inputs.runner_image_ref != '' + if: needs.prepare-backend.outputs.ref != '' && needs.prepare-runner.outputs.ref != '' && needs.prepare-frontend.outputs.ref != '' uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 with: registry: ghcr.io @@ -89,10 +248,11 @@ jobs: - name: Boot docker-compose stack run: | - if [ -n "$BACKEND_IMAGE" ] && [ -n "$RUNNER_IMAGE" ]; then + set -euo pipefail + if [ -n "$BACKEND_IMAGE" ] && [ -n "$RUNNER_IMAGE" ] && [ -n "$FRONTEND_IMAGE" ]; then docker pull "$BACKEND_IMAGE" docker pull "$RUNNER_IMAGE" - docker compose build frontend + docker pull "$FRONTEND_IMAGE" docker compose up -d --no-build backend frontend else docker compose up -d backend frontend @@ -161,6 +321,7 @@ jobs: e2e: name: Playwright (chromium) + needs: [prepare-backend, prepare-runner, prepare-frontend] runs-on: ubuntu-latest # Sharding splits the exhaustive suite across isolated runners. The # capacity record owns the measured decision and the shard-1 guard below @@ -223,13 +384,14 @@ jobs: MAX_SESSIONS_PER_USER: "60" MAX_SESSIONS_GLOBAL: "200" DOCKER_EXEC_CONCURRENCY: "16" - BACKEND_IMAGE: ${{ inputs.backend_image_ref || '' }} - RUNNER_IMAGE: ${{ inputs.runner_image_ref || '' }} + BACKEND_IMAGE: ${{ needs.prepare-backend.outputs.ref }} + RUNNER_IMAGE: ${{ needs.prepare-runner.outputs.ref }} + FRONTEND_IMAGE: ${{ needs.prepare-frontend.outputs.ref }} steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Log in to GHCR for immutable candidates - if: inputs.backend_image_ref != '' && inputs.runner_image_ref != '' + if: needs.prepare-backend.outputs.ref != '' && needs.prepare-runner.outputs.ref != '' && needs.prepare-frontend.outputs.ref != '' uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 with: registry: ghcr.io @@ -247,10 +409,11 @@ jobs: - name: Boot docker-compose stack run: | - if [ -n "$BACKEND_IMAGE" ] && [ -n "$RUNNER_IMAGE" ]; then + set -euo pipefail + if [ -n "$BACKEND_IMAGE" ] && [ -n "$RUNNER_IMAGE" ] && [ -n "$FRONTEND_IMAGE" ]; then docker pull "$BACKEND_IMAGE" docker pull "$RUNNER_IMAGE" - docker compose build frontend + docker pull "$FRONTEND_IMAGE" docker compose up -d --no-build backend frontend else docker compose up -d backend frontend @@ -324,6 +487,7 @@ jobs: cross-browser-core: name: Playwright core (${{ matrix.browser }}) + needs: [prepare-backend, prepare-runner, prepare-frontend] runs-on: ubuntu-latest timeout-minutes: 15 strategy: @@ -343,13 +507,14 @@ jobs: MAX_SESSIONS_PER_USER: "20" MAX_SESSIONS_GLOBAL: "80" DOCKER_EXEC_CONCURRENCY: "8" - BACKEND_IMAGE: ${{ inputs.backend_image_ref || '' }} - RUNNER_IMAGE: ${{ inputs.runner_image_ref || '' }} + BACKEND_IMAGE: ${{ needs.prepare-backend.outputs.ref }} + RUNNER_IMAGE: ${{ needs.prepare-runner.outputs.ref }} + FRONTEND_IMAGE: ${{ needs.prepare-frontend.outputs.ref }} steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Log in to GHCR for immutable candidates - if: inputs.backend_image_ref != '' && inputs.runner_image_ref != '' + if: needs.prepare-backend.outputs.ref != '' && needs.prepare-runner.outputs.ref != '' && needs.prepare-frontend.outputs.ref != '' uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 with: registry: ghcr.io @@ -367,10 +532,11 @@ jobs: - name: Boot docker-compose stack run: | - if [ -n "$BACKEND_IMAGE" ] && [ -n "$RUNNER_IMAGE" ]; then + set -euo pipefail + if [ -n "$BACKEND_IMAGE" ] && [ -n "$RUNNER_IMAGE" ] && [ -n "$FRONTEND_IMAGE" ]; then docker pull "$BACKEND_IMAGE" docker pull "$RUNNER_IMAGE" - docker compose build frontend + docker pull "$FRONTEND_IMAGE" docker compose up -d --no-build backend frontend else docker compose up -d backend frontend @@ -474,3 +640,42 @@ jobs: path: e2e/shadow-results retention-days: 90 if-no-files-found: error + + cleanup-images: + name: Clean temporary E2E images + if: >- + always() && + needs.e2e.result == 'success' && + needs.cross-browser-core.result == 'success' && + (needs.prepare-backend.outputs.created == 'true' || + needs.prepare-runner.outputs.created == 'true' || + needs.prepare-frontend.outputs.created == 'true') + needs: + - prepare-backend + - prepare-runner + - prepare-frontend + - critical-shadow + - e2e + - cross-browser-core + - shadow-evidence + runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ github.token }} + RUN_TAG: run-${{ github.run_id }} + BACKEND_CREATED: ${{ needs.prepare-backend.outputs.created }} + RUNNER_CREATED: ${{ needs.prepare-runner.outputs.created }} + FRONTEND_CREATED: ${{ needs.prepare-frontend.outputs.created }} + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - name: Delete exact run-scoped package versions + run: | + set -euo pipefail + if [ "$BACKEND_CREATED" = "true" ]; then + node .github/scripts/ghcr-retention.mjs --owner "$GITHUB_REPOSITORY_OWNER" --package codetutor-ci-backend --tag "$RUN_TAG" --require-match + fi + if [ "$RUNNER_CREATED" = "true" ]; then + node .github/scripts/ghcr-retention.mjs --owner "$GITHUB_REPOSITORY_OWNER" --package codetutor-ci-runner --tag "$RUN_TAG" --require-match + fi + if [ "$FRONTEND_CREATED" = "true" ]; then + node .github/scripts/ghcr-retention.mjs --owner "$GITHUB_REPOSITORY_OWNER" --package codetutor-ci-frontend --tag "$RUN_TAG" --require-match + fi