Standardize configurable QA workflows for the trial

.github/workflows/ci-check.yml

@@ -18,4 +18,4 @@ jobs:
     uses: ./.github/workflows/check.yml
     with:
       install: false
-      command: node --test tests/ready.test.mjs
+      command: node --test tests/*.test.mjs

.github/workflows/ci-ready.yml

@@ -9,6 +9,7 @@ on:
     types: [completed]
 
 permissions:
+  actions: read
   contents: write
   pull-requests: write
   statuses: write

.github/workflows/ready.yml

@@ -12,7 +12,12 @@ on:
       check-name:
         type: string
         default: Check / Check
-
+      require-coderabbit:
+        type: boolean
+        default: true
+      test-checks:
+        type: string
+        default: "[]"
       merge-app-client-id:
         type: string
         default: ""
@@ -21,6 +26,7 @@ on:
         required: false
 
 permissions:
+  actions: read
   contents: write
   pull-requests: write
   statuses: write
@@ -47,11 +53,13 @@ jobs:
           repositories: ${{ github.event.repository.name }}
       # Only the merge uses this token, so existing push-triggered releases still run.
       # Only GitHub metadata is read. Never check out or execute PR code here.
-      - name: Require selected review and publish CloudCannon changes
+      - name: Reconcile publication requirements
         uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
         env:
           NETLIFY_CHECKS: ${{ inputs.netlify-checks }}
           CHECK_NAME: ${{ inputs.check-name }}
+          REQUIRE_CODERABBIT: ${{ inputs.require-coderabbit }}
+          TEST_CHECKS: ${{ inputs.test-checks }}
           MERGE_TOKEN: ${{ steps.merge-token.outputs.token }}
         with:
           script: |
@@ -75,7 +83,52 @@ jobs:
               const pull = await getPull();
               if (pull.state !== 'open' || pull.base.ref !== repository.default_branch) return;
               const sha = pull.head.sha;
-              const selected = pull.labels.some(label => label.name === 'review-ready');
+              const selected = process.env.REQUIRE_CODERABBIT !== 'false'
+                && pull.labels.some(label => label.name === 'review-ready');
+              const requiredTests = JSON.parse(process.env.TEST_CHECKS || '[]');
+              const isSelected = candidate => process.env.REQUIRE_CODERABBIT !== 'false'
+                && candidate.labels.some(label => label.name === 'review-ready');
+              if (requiredTests.length) await github.rest.repos.createCommitStatus({owner, repo, sha,
+                context: 'Review', state: 'pending', description: 'Checking the current Test run and attempt.',
+                target_url: pull.html_url});
+              async function testsPassed() {
+                if (!requiredTests.length) return true;
+                // Read the current run and attempt on every event. Check runs can
+                // still show a previous success while the workflow is rerunning.
+                const event = context.payload.workflow_run;
+                let completedRunPending = false;
+                for (let retry = 0; retry < 4; retry++) {
+                  const runs = await github.paginate(github.rest.actions.listWorkflowRunsForRepo,
+                    {owner, repo, head_sha: sha, event: 'pull_request', per_page: 100});
+                  const run = runs.filter(run => run.name === 'Test' && run.head_sha === sha
+                    && run.path === '.github/workflows/test.yml')
+                    .sort((a, b) => b.id - a.id)[0];
+                  const eventIsNewer = event?.name === 'Test' && event.head_sha === sha
+                    && (!run || event.id > run.id || (event.id === run.id
+                      && event.run_attempt > run.run_attempt));
+                  const eventIsCurrent = event?.name === 'Test' && event.head_sha === sha
+                    && run && event.id === run.id && event.run_attempt === run.run_attempt;
+                  const completedEvent = (eventIsNewer || eventIsCurrent)
+                    && event.status === 'completed' && event.conclusion === 'success';
+                  completedRunPending = completedEvent
+                    || (run?.status === 'completed' && run.conclusion === 'success');
+                  if (eventIsNewer) {
+                    if (!completedEvent) return false;
+                  } else if (run) {
+                    if (run.status === 'completed' && run.conclusion === 'success') {
+                      const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRunAttempt,
+                        {owner, repo, run_id: run.id, attempt_number: run.run_attempt, per_page: 100});
+                      if (requiredTests.every(name => jobs.some(job => job.name === name
+                        && job.status === 'completed' && job.conclusion === 'success'))) return true;
+                    } else if (!completedEvent) return false;
+                  }
+                  // Successful completion can precede its jobs' metadata. Retry
+                  // briefly so a final completion event cannot strand the PR.
+                  if (retry < 3) await new Promise(resolve => setTimeout(resolve, 2000));
+                }
+                if (completedRunPending) throw new Error('Successful Test metadata is still incomplete. Rerun Ready to reconcile it.');
+                return false;
+              }
               async function hasApproval() {
                 if (!selected) return true;
                 const reviews = await github.paginate(github.rest.pulls.listReviews, {owner, repo, pull_number: number, per_page: 100});
@@ -102,16 +155,17 @@ jobs:
                   throw error;
                 }
               }
-              const approved = await hasTrustedNotifier() && await hasApproval();
+              const approved = await hasTrustedNotifier() && await hasApproval() && await testsPassed();
               const current = await getPull();
               if (current.base.ref !== pull.base.ref || current.head.sha !== sha || current.draft !== pull.draft
-                || current.labels.some(label => label.name === 'review-ready') !== selected) return;
-              const reviewPassed = !pull.draft && approved && await hasTrustedNotifier() && await hasApproval();
+                || isSelected(current) !== selected) return;
+              const reviewPassed = !pull.draft && approved && await hasTrustedNotifier() && await hasApproval()
+                && await testsPassed();
               await github.rest.repos.createCommitStatus({owner, repo, sha, context: 'Review',
                 state: reviewPassed ? 'success' : 'pending',
                 description: pull.draft ? 'Draft: publication is not authorized.'
-                  : approved ? (selected ? 'CodeRabbit approved this revision.' : 'No CodeRabbit review selected.')
-                  : 'Waiting for CodeRabbit approval of this revision.',
+                  : reviewPassed ? (selected ? 'CodeRabbit approved this revision.' : 'Ready requirements passed for this revision.')
+                  : 'Waiting for required tests or selected review of this revision.',
                 target_url: pull.html_url});
               if (!reviewPassed) return;
               // Agents own code merges so they can follow the production deployment.
@@ -142,13 +196,18 @@ jobs:
               if (!repository.allow_auto_merge) return core.info('Repository auto-merge is not enabled yet.');
               const final = await getPull();
               if (final.state !== 'open' || final.draft || final.base.ref !== pull.base.ref || final.head.sha !== sha
-                || final.labels.some(label => label.name === 'review-ready') !== selected) return;
+                || isSelected(final) !== selected) return;
               if (!isCloudCannonPublication(final)) return;
               if (selected && (!await hasTrustedNotifier() || !await hasApproval())) {
                 await github.rest.repos.createCommitStatus({owner, repo, sha, context: 'Review',
                   state: 'pending', description: 'CodeRabbit approval changed; waiting for review.', target_url: pull.html_url});
                 return;
               }
+              if (!await testsPassed()) {
+                await github.rest.repos.createCommitStatus({owner, repo, sha, context: 'Review',
+                  state: 'pending', description: 'Waiting for required tests of this revision.', target_url: pull.html_url});
+                return;
+              }
               const method = final.head.ref === 'preview' && final.head.repo?.full_name === final.base.repo?.full_name
                 ? 'merge' : 'squash';
               // Unrequired failed jobs can make GitHub report unstable; required rules still enforce this merge.

.github/workflows/test-live.yml

@@ -0,0 +1,58 @@
+name: Test live
+
+on:
+  workflow_call:
+    inputs:
+      runner:
+        type: string
+        default: ubuntu-latest
+      base-url:
+        type: string
+        required: true
+      revision:
+        description: Commit confirmed as published before dispatching this workflow.
+        type: string
+        required: true
+      http:
+        type: boolean
+        default: true
+      browser:
+        type: boolean
+        default: true
+
+permissions:
+  contents: read
+
+jobs:
+  live:
+    name: Live
+    runs-on: ${{ inputs.runner }}
+    timeout-minutes: 15
+    env:
+      pnpm_config_ignore_scripts: "true"
+      pnpm_config_ignore_pnpmfile: "true"
+      BASE_URL: ${{ inputs.base-url }}
+      QA_PROFILE: live
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ inputs.revision }}
+          persist-credentials: false
+      - uses: voidzero-dev/setup-vp@68f39c78aaf39be468cabd2afb09c162574c84af # v1.21.2
+        with:
+          version: "1.1.0"
+          node-version-file: .node-version
+          cache: true
+          run-install: |
+            args:
+              - --frozen-lockfile
+              - --ignore-scripts
+      - name: Safe live HTTP smoke
+        if: ${{ inputs.http }}
+        run: vp run test:http
+      - name: Install desktop browsers
+        if: ${{ inputs.browser }}
+        run: vp exec playwright install --with-deps chromium firefox webkit
+      - name: Safe live browser smoke
+        if: ${{ inputs.browser }}
+        run: vp run test:browser

.github/workflows/test.yml

@@ -0,0 +1,173 @@
+name: Test
+
+on:
+  workflow_call:
+    inputs:
+      runner:
+        type: string
+        default: ubuntu-latest
+      unit:
+        type: boolean
+        default: false
+      http:
+        type: boolean
+        default: false
+      browser:
+        type: boolean
+        default: false
+      preview-checks:
+        description: JSON array of Netlify status contexts; the first is BASE_URL.
+        type: string
+        default: "[]"
+      node-version-file:
+        type: string
+        default: .node-version
+    secrets:
+      testing-secret:
+        required: false
+
+permissions:
+  contents: read
+  statuses: read
+  pull-requests: read
+
+jobs:
+  unit:
+    name: Unit
+    if: ${{ !github.event.pull_request.draft && inputs.unit }}
+    runs-on: ${{ inputs.runner }}
+    timeout-minutes: 20
+    env:
+      pnpm_config_ignore_scripts: "true"
+      pnpm_config_ignore_pnpmfile: "true"
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: voidzero-dev/setup-vp@68f39c78aaf39be468cabd2afb09c162574c84af # v1.21.2
+        with:
+          version: "1.1.0"
+          node-version-file: ${{ inputs.node-version-file }}
+          cache: true
+          run-install: |
+            args:
+              - --frozen-lockfile
+              - --ignore-scripts
+      - name: Unit and service tests
+        run: vp run test:unit
+
+  web:
+    name: Web
+    if: ${{ !github.event.pull_request.draft && (inputs.http || inputs.browser) }}
+    runs-on: ${{ inputs.runner }}
+    timeout-minutes: 30
+    # Signed-in tests can share preview accounts across browser targets and PRs.
+    concurrency:
+      group: qa-web-${{ github.repository }}
+      cancel-in-progress: false
+    env:
+      pnpm_config_ignore_scripts: "true"
+      pnpm_config_ignore_pnpmfile: "true"
+      QA_PROFILE: preview
+    steps:
+      - name: Resolve previews for this PR revision
+        id: preview
+        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
+        env:
+          PREVIEW_CHECKS: ${{ inputs.preview-checks }}
+        with:
+          script: |
+            const {owner, repo} = context.repo;
+            const pull = context.payload.pull_request;
+            if (!pull) throw new Error('URL tests require a pull_request event.');
+            const names = JSON.parse(process.env.PREVIEW_CHECKS);
+            if (!Array.isArray(names) || !names.length
+              || names.some(name => typeof name !== 'string' || !/^netlify\/[a-z0-9-]+\/deploy-preview$/.test(name))) {
+              throw new Error('Enabled URL tests require preview-checks.');
+            }
+            const sha = pull.head.sha;
+            const deadline = Date.now() + 12 * 60 * 1000;
+            while (Date.now() < deadline) {
+              const {data: current} = await github.rest.pulls.get({owner, repo, pull_number: pull.number});
+              if (current.state !== 'open' || current.draft || current.head.sha !== sha) {
+                throw new Error('PR changed or returned to draft; discard this test run.');
+              }
+              const {data} = await github.rest.repos.getCombinedStatusForRef({owner, repo, ref: sha, per_page: 100});
+              const urls = {};
+              for (const name of names) {
+                const status = data.statuses.find(status => status.context === name);
+                if (['error', 'failure'].includes(status?.state)) {
+                  throw new Error(`${name} failed for ${sha}.`);
+                }
+                if (status?.state !== 'success') continue;
+                const url = new URL(status.target_url);
+                const site = name.split('/')[1];
+                if (url.protocol !== 'https:' || url.username || url.password
+                  || url.hostname !== `deploy-preview-${pull.number}--${site}.netlify.app`) {
+                  throw new Error(`${name} did not return this PR's Netlify preview URL.`);
+                }
+                urls[name] = url.origin;
+              }
+              if (names.every(name => urls[name])) {
+                core.setOutput('base-url', urls[names[0]]);
+                core.setOutput('deploy-urls', JSON.stringify(urls));
+                core.setOutput('sha', sha);
+                return;
+              }
+              core.info(`Waiting for matching previews: ${names.filter(name => !urls[name]).join(', ')}`);
+              await new Promise(resolve => setTimeout(resolve, 15000));
+            }
+            throw new Error('Matching previews did not complete within 12 minutes.');
+      - name: Checkout the tested preview revision
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ steps.preview.outputs.sha }}
+          persist-credentials: false
+      - uses: voidzero-dev/setup-vp@68f39c78aaf39be468cabd2afb09c162574c84af # v1.21.2
+        with:
+          version: "1.1.0"
+          node-version-file: ${{ inputs.node-version-file }}
+          cache: true
+          run-install: |
+            args:
+              - --frozen-lockfile
+              - --ignore-scripts
+      - name: HTTP tests
+        if: ${{ inputs.http }}
+        env:
+          BASE_URL: ${{ steps.preview.outputs.base-url }}
+          QA_DEPLOY_URLS: ${{ steps.preview.outputs.deploy-urls }}
+        run: vp run test:http
+      - name: Install desktop browsers
+        if: ${{ inputs.browser }}
+        run: vp exec playwright install --with-deps chromium firefox webkit
+      - name: Browser tests
+        if: ${{ inputs.browser }}
+        env:
+          BASE_URL: ${{ steps.preview.outputs.base-url }}
+          QA_DEPLOY_URLS: ${{ steps.preview.outputs.deploy-urls }}
+          TESTING_SECRET: ${{ secrets.testing-secret }}
+        run: vp run test:browser
+
+  test:
+    name: Test
+    if: ${{ always() && !github.event.pull_request.draft }}
+    needs: [unit, web]
+    runs-on: ${{ inputs.runner }}
+    timeout-minutes: 2
+    steps:
+      - name: Require every enabled category
+        env:
+          UNIT_ENABLED: ${{ inputs.unit }}
+          WEB_ENABLED: ${{ inputs.http || inputs.browser }}
+          UNIT_RESULT: ${{ needs.unit.result }}
+          WEB_RESULT: ${{ needs.web.result }}
+        run: |
+          if [[ "$UNIT_ENABLED" == true && "$UNIT_RESULT" != success ]]; then
+            echo 'Enabled unit tests did not succeed.' >&2
+            exit 1
+          fi
+          if [[ "$WEB_ENABLED" == true && "$WEB_RESULT" != success ]]; then
+            echo 'Enabled URL tests did not succeed.' >&2
+            exit 1
+          fi