diff --git a/.github/workflows/ci-check.yml b/.github/workflows/ci-check.yml index a428eb6..8a34fc9 100644 --- a/.github/workflows/ci-check.yml +++ b/.github/workflows/ci-check.yml @@ -18,4 +18,4 @@ jobs: uses: ./.github/workflows/check.yml with: install: false - command: node --test tests/ready.test.mjs + command: node --test tests/*.test.mjs diff --git a/.github/workflows/ci-ready.yml b/.github/workflows/ci-ready.yml index 351cfe6..654c643 100644 --- a/.github/workflows/ci-ready.yml +++ b/.github/workflows/ci-ready.yml @@ -9,6 +9,7 @@ on: types: [completed] permissions: + actions: read contents: write pull-requests: write statuses: write diff --git a/.github/workflows/ready.yml b/.github/workflows/ready.yml index 8513e71..15f1598 100644 --- a/.github/workflows/ready.yml +++ b/.github/workflows/ready.yml @@ -12,7 +12,12 @@ on: check-name: type: string default: Check / Check - + require-coderabbit: + type: boolean + default: true + test-checks: + type: string + default: "[]" merge-app-client-id: type: string default: "" @@ -21,6 +26,7 @@ on: required: false permissions: + actions: read contents: write pull-requests: write statuses: write @@ -47,11 +53,13 @@ jobs: repositories: ${{ github.event.repository.name }} # Only the merge uses this token, so existing push-triggered releases still run. # Only GitHub metadata is read. Never check out or execute PR code here. - - name: Require selected review and publish CloudCannon changes + - name: Reconcile publication requirements uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 env: NETLIFY_CHECKS: ${{ inputs.netlify-checks }} CHECK_NAME: ${{ inputs.check-name }} + REQUIRE_CODERABBIT: ${{ inputs.require-coderabbit }} + TEST_CHECKS: ${{ inputs.test-checks }} MERGE_TOKEN: ${{ steps.merge-token.outputs.token }} with: script: | @@ -75,7 +83,52 @@ jobs: const pull = await getPull(); if (pull.state !== 'open' || pull.base.ref !== repository.default_branch) return; const sha = pull.head.sha; - const selected = pull.labels.some(label => label.name === 'review-ready'); + const selected = process.env.REQUIRE_CODERABBIT !== 'false' + && pull.labels.some(label => label.name === 'review-ready'); + const requiredTests = JSON.parse(process.env.TEST_CHECKS || '[]'); + const isSelected = candidate => process.env.REQUIRE_CODERABBIT !== 'false' + && candidate.labels.some(label => label.name === 'review-ready'); + if (requiredTests.length) await github.rest.repos.createCommitStatus({owner, repo, sha, + context: 'Review', state: 'pending', description: 'Checking the current Test run and attempt.', + target_url: pull.html_url}); + async function testsPassed() { + if (!requiredTests.length) return true; + // Read the current run and attempt on every event. Check runs can + // still show a previous success while the workflow is rerunning. + const event = context.payload.workflow_run; + let completedRunPending = false; + for (let retry = 0; retry < 4; retry++) { + const runs = await github.paginate(github.rest.actions.listWorkflowRunsForRepo, + {owner, repo, head_sha: sha, event: 'pull_request', per_page: 100}); + const run = runs.filter(run => run.name === 'Test' && run.head_sha === sha + && run.path === '.github/workflows/test.yml') + .sort((a, b) => b.id - a.id)[0]; + const eventIsNewer = event?.name === 'Test' && event.head_sha === sha + && (!run || event.id > run.id || (event.id === run.id + && event.run_attempt > run.run_attempt)); + const eventIsCurrent = event?.name === 'Test' && event.head_sha === sha + && run && event.id === run.id && event.run_attempt === run.run_attempt; + const completedEvent = (eventIsNewer || eventIsCurrent) + && event.status === 'completed' && event.conclusion === 'success'; + completedRunPending = completedEvent + || (run?.status === 'completed' && run.conclusion === 'success'); + if (eventIsNewer) { + if (!completedEvent) return false; + } else if (run) { + if (run.status === 'completed' && run.conclusion === 'success') { + const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRunAttempt, + {owner, repo, run_id: run.id, attempt_number: run.run_attempt, per_page: 100}); + if (requiredTests.every(name => jobs.some(job => job.name === name + && job.status === 'completed' && job.conclusion === 'success'))) return true; + } else if (!completedEvent) return false; + } + // Successful completion can precede its jobs' metadata. Retry + // briefly so a final completion event cannot strand the PR. + if (retry < 3) await new Promise(resolve => setTimeout(resolve, 2000)); + } + if (completedRunPending) throw new Error('Successful Test metadata is still incomplete. Rerun Ready to reconcile it.'); + return false; + } async function hasApproval() { if (!selected) return true; const reviews = await github.paginate(github.rest.pulls.listReviews, {owner, repo, pull_number: number, per_page: 100}); @@ -102,16 +155,17 @@ jobs: throw error; } } - const approved = await hasTrustedNotifier() && await hasApproval(); + const approved = await hasTrustedNotifier() && await hasApproval() && await testsPassed(); const current = await getPull(); if (current.base.ref !== pull.base.ref || current.head.sha !== sha || current.draft !== pull.draft - || current.labels.some(label => label.name === 'review-ready') !== selected) return; - const reviewPassed = !pull.draft && approved && await hasTrustedNotifier() && await hasApproval(); + || isSelected(current) !== selected) return; + const reviewPassed = !pull.draft && approved && await hasTrustedNotifier() && await hasApproval() + && await testsPassed(); await github.rest.repos.createCommitStatus({owner, repo, sha, context: 'Review', state: reviewPassed ? 'success' : 'pending', description: pull.draft ? 'Draft: publication is not authorized.' - : approved ? (selected ? 'CodeRabbit approved this revision.' : 'No CodeRabbit review selected.') - : 'Waiting for CodeRabbit approval of this revision.', + : reviewPassed ? (selected ? 'CodeRabbit approved this revision.' : 'Ready requirements passed for this revision.') + : 'Waiting for required tests or selected review of this revision.', target_url: pull.html_url}); if (!reviewPassed) return; // Agents own code merges so they can follow the production deployment. @@ -142,13 +196,18 @@ jobs: if (!repository.allow_auto_merge) return core.info('Repository auto-merge is not enabled yet.'); const final = await getPull(); if (final.state !== 'open' || final.draft || final.base.ref !== pull.base.ref || final.head.sha !== sha - || final.labels.some(label => label.name === 'review-ready') !== selected) return; + || isSelected(final) !== selected) return; if (!isCloudCannonPublication(final)) return; if (selected && (!await hasTrustedNotifier() || !await hasApproval())) { await github.rest.repos.createCommitStatus({owner, repo, sha, context: 'Review', state: 'pending', description: 'CodeRabbit approval changed; waiting for review.', target_url: pull.html_url}); return; } + if (!await testsPassed()) { + await github.rest.repos.createCommitStatus({owner, repo, sha, context: 'Review', + state: 'pending', description: 'Waiting for required tests of this revision.', target_url: pull.html_url}); + return; + } const method = final.head.ref === 'preview' && final.head.repo?.full_name === final.base.repo?.full_name ? 'merge' : 'squash'; // Unrequired failed jobs can make GitHub report unstable; required rules still enforce this merge. diff --git a/.github/workflows/test-live.yml b/.github/workflows/test-live.yml new file mode 100644 index 0000000..51a6f2e --- /dev/null +++ b/.github/workflows/test-live.yml @@ -0,0 +1,58 @@ +name: Test live + +on: + workflow_call: + inputs: + runner: + type: string + default: ubuntu-latest + base-url: + type: string + required: true + revision: + description: Commit confirmed as published before dispatching this workflow. + type: string + required: true + http: + type: boolean + default: true + browser: + type: boolean + default: true + +permissions: + contents: read + +jobs: + live: + name: Live + runs-on: ${{ inputs.runner }} + timeout-minutes: 15 + env: + pnpm_config_ignore_scripts: "true" + pnpm_config_ignore_pnpmfile: "true" + BASE_URL: ${{ inputs.base-url }} + QA_PROFILE: live + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.revision }} + persist-credentials: false + - uses: voidzero-dev/setup-vp@68f39c78aaf39be468cabd2afb09c162574c84af # v1.21.2 + with: + version: "1.1.0" + node-version-file: .node-version + cache: true + run-install: | + args: + - --frozen-lockfile + - --ignore-scripts + - name: Safe live HTTP smoke + if: ${{ inputs.http }} + run: vp run test:http + - name: Install desktop browsers + if: ${{ inputs.browser }} + run: vp exec playwright install --with-deps chromium firefox webkit + - name: Safe live browser smoke + if: ${{ inputs.browser }} + run: vp run test:browser diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..26ffaa9 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,173 @@ +name: Test + +on: + workflow_call: + inputs: + runner: + type: string + default: ubuntu-latest + unit: + type: boolean + default: false + http: + type: boolean + default: false + browser: + type: boolean + default: false + preview-checks: + description: JSON array of Netlify status contexts; the first is BASE_URL. + type: string + default: "[]" + node-version-file: + type: string + default: .node-version + secrets: + testing-secret: + required: false + +permissions: + contents: read + statuses: read + pull-requests: read + +jobs: + unit: + name: Unit + if: ${{ !github.event.pull_request.draft && inputs.unit }} + runs-on: ${{ inputs.runner }} + timeout-minutes: 20 + env: + pnpm_config_ignore_scripts: "true" + pnpm_config_ignore_pnpmfile: "true" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: voidzero-dev/setup-vp@68f39c78aaf39be468cabd2afb09c162574c84af # v1.21.2 + with: + version: "1.1.0" + node-version-file: ${{ inputs.node-version-file }} + cache: true + run-install: | + args: + - --frozen-lockfile + - --ignore-scripts + - name: Unit and service tests + run: vp run test:unit + + web: + name: Web + if: ${{ !github.event.pull_request.draft && (inputs.http || inputs.browser) }} + runs-on: ${{ inputs.runner }} + timeout-minutes: 30 + # Signed-in tests can share preview accounts across browser targets and PRs. + concurrency: + group: qa-web-${{ github.repository }} + cancel-in-progress: false + env: + pnpm_config_ignore_scripts: "true" + pnpm_config_ignore_pnpmfile: "true" + QA_PROFILE: preview + steps: + - name: Resolve previews for this PR revision + id: preview + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + PREVIEW_CHECKS: ${{ inputs.preview-checks }} + with: + script: | + const {owner, repo} = context.repo; + const pull = context.payload.pull_request; + if (!pull) throw new Error('URL tests require a pull_request event.'); + const names = JSON.parse(process.env.PREVIEW_CHECKS); + if (!Array.isArray(names) || !names.length + || names.some(name => typeof name !== 'string' || !/^netlify\/[a-z0-9-]+\/deploy-preview$/.test(name))) { + throw new Error('Enabled URL tests require preview-checks.'); + } + const sha = pull.head.sha; + const deadline = Date.now() + 12 * 60 * 1000; + while (Date.now() < deadline) { + const {data: current} = await github.rest.pulls.get({owner, repo, pull_number: pull.number}); + if (current.state !== 'open' || current.draft || current.head.sha !== sha) { + throw new Error('PR changed or returned to draft; discard this test run.'); + } + const {data} = await github.rest.repos.getCombinedStatusForRef({owner, repo, ref: sha, per_page: 100}); + const urls = {}; + for (const name of names) { + const status = data.statuses.find(status => status.context === name); + if (['error', 'failure'].includes(status?.state)) { + throw new Error(`${name} failed for ${sha}.`); + } + if (status?.state !== 'success') continue; + const url = new URL(status.target_url); + const site = name.split('/')[1]; + if (url.protocol !== 'https:' || url.username || url.password + || url.hostname !== `deploy-preview-${pull.number}--${site}.netlify.app`) { + throw new Error(`${name} did not return this PR's Netlify preview URL.`); + } + urls[name] = url.origin; + } + if (names.every(name => urls[name])) { + core.setOutput('base-url', urls[names[0]]); + core.setOutput('deploy-urls', JSON.stringify(urls)); + core.setOutput('sha', sha); + return; + } + core.info(`Waiting for matching previews: ${names.filter(name => !urls[name]).join(', ')}`); + await new Promise(resolve => setTimeout(resolve, 15000)); + } + throw new Error('Matching previews did not complete within 12 minutes.'); + - name: Checkout the tested preview revision + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ steps.preview.outputs.sha }} + persist-credentials: false + - uses: voidzero-dev/setup-vp@68f39c78aaf39be468cabd2afb09c162574c84af # v1.21.2 + with: + version: "1.1.0" + node-version-file: ${{ inputs.node-version-file }} + cache: true + run-install: | + args: + - --frozen-lockfile + - --ignore-scripts + - name: HTTP tests + if: ${{ inputs.http }} + env: + BASE_URL: ${{ steps.preview.outputs.base-url }} + QA_DEPLOY_URLS: ${{ steps.preview.outputs.deploy-urls }} + run: vp run test:http + - name: Install desktop browsers + if: ${{ inputs.browser }} + run: vp exec playwright install --with-deps chromium firefox webkit + - name: Browser tests + if: ${{ inputs.browser }} + env: + BASE_URL: ${{ steps.preview.outputs.base-url }} + QA_DEPLOY_URLS: ${{ steps.preview.outputs.deploy-urls }} + TESTING_SECRET: ${{ secrets.testing-secret }} + run: vp run test:browser + + test: + name: Test + if: ${{ always() && !github.event.pull_request.draft }} + needs: [unit, web] + runs-on: ${{ inputs.runner }} + timeout-minutes: 2 + steps: + - name: Require every enabled category + env: + UNIT_ENABLED: ${{ inputs.unit }} + WEB_ENABLED: ${{ inputs.http || inputs.browser }} + UNIT_RESULT: ${{ needs.unit.result }} + WEB_RESULT: ${{ needs.web.result }} + run: | + if [[ "$UNIT_ENABLED" == true && "$UNIT_RESULT" != success ]]; then + echo 'Enabled unit tests did not succeed.' >&2 + exit 1 + fi + if [[ "$WEB_ENABLED" == true && "$WEB_RESULT" != success ]]; then + echo 'Enabled URL tests did not succeed.' >&2 + exit 1 + fi diff --git a/README.md b/README.md index 188b3b6..4ad5f90 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,20 @@ # fulldotdev/.github -Organization-wide GitHub defaults and reusable workflows. Repository callers select their own check command; workflows here are not inherited automatically. +Organization-wide GitHub defaults and reusable workflows. Repositories own their package scripts, assertions and enabled categories. These workflows own execution and time limits; they are not inherited automatically. Callers pin a full commit SHA from `main` after the shared workflow PR is merged. New projects take their callers from [astro-template](https://github.com/fulldotdev/astro-template) or [vite-template](https://github.com/fulldotdev/vite-template). -- **Finalize (Prepare)** leaves a complete draft PR. Previews build; full checks wait for Ready. -- **Dispatch (Release)** validates the affected behavior and marks the PR Ready. After the required checks and previews pass, the agent merges the validated head and follows its production deployment. Choose `review-ready` before marking Ready when CodeRabbit approval is needed. -- **Settle (Close)** follows production verification and preserves the result before closing the task. +- **Develop** stays local while implementing and experimenting. +- **Draft PR** builds and checks artifacts locally, then opens a shared draft and Netlify preview. +- **Make PR ready** runs enabled checks and tests locally, then CI confirms them. Source checks and unit tests run alongside the Netlify build. URL tests use the preview of the tested PR head. +- **Merge PR** merges the validated head and follows the production build and short, safe live checks. -Organization rules require `Check / Check` and `Review`, without requiring synchronization with main. Netlify requirements are managed centrally for the repositories with those sites. Production keeps its scoped check and build. A skipped or outdated CodeRabbit review cannot satisfy a selected review. +The trial callers in `fulldev`, `astro-template` and `vite-template` use `check:source` and the selected categories in `test.yml`. Devices stay manual. They pass `require-coderabbit: false` and require `Test / Test` through the existing `Review` status. Missing, skipped, failed or superseded test results cannot pass that gate. Other callers retain their existing behavior until migrated. + +The Ready caller grants `actions: read` so the gate reads the current Test workflow attempt and its jobs, including reruns. Existing repositories remain pinned to their previous workflow revision until migrated. + +Organization rules require `Check / Check` and `Review`, without requiring synchronization with main. Netlify requirements are managed centrally for repositories with those sites. Production keeps scoped source, build and artifact checks. Live URL tests run after the current production deployment has been confirmed, rather than inside its build command. Keep repository auto-merge disabled except for CloudCannon publishers. CloudCannon's Publish button keeps automatic publication for same-repository `preview` PRs created by `cloudcannon[bot]`, with all required checks and previews. Code PRs are merged by their agent. diff --git a/tests/preview.test.mjs b/tests/preview.test.mjs new file mode 100644 index 0000000..66c4b7d --- /dev/null +++ b/tests/preview.test.mjs @@ -0,0 +1,64 @@ +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import test from 'node:test'; + +const source = readFileSync(new URL('../.github/workflows/test.yml', import.meta.url), 'utf8') + .split(' script: |\n')[1].split('\n - name: Checkout')[0] + .split('\n').map(line => line.slice(12)).join('\n'); +const execute = new (Object.getPrototypeOf(async function () {}).constructor)( + 'github', 'context', 'core', 'process', 'Date', 'setTimeout', source); + +async function scenario(options = {}) { + const pull = {number: 7, state: 'open', draft: false, head: {sha: 'tested-head'}}; + const names = options.names ?? ['netlify/app/deploy-preview']; + const outputs = {}; + const refs = []; + let clock = 0; + const github = {rest: { + pulls: {get: async () => ({data: {...pull, ...options.current}})}, + repos: {getCombinedStatusForRef: async ({ref}) => { + refs.push(ref); + return {data: {statuses: options.statuses ?? names.map(context => ({context, state: 'success', + target_url: `https://deploy-preview-7--${context.split('/')[1]}.netlify.app`}))}}; + }}, + }}; + await execute(github, {repo: {owner: 'o', repo: 'r'}, payload: {pull_request: pull}}, + {info() {}, setOutput: (name, value) => { outputs[name] = value; }}, + {env: {PREVIEW_CHECKS: JSON.stringify(names)}}, + {now: () => clock}, callback => { clock += 60 * 1000; callback(); }); + return {outputs, refs}; +} + +test('URL tests use previews associated with the exact PR head', async () => { + const {outputs, refs} = await scenario({names: ['netlify/app/deploy-preview', 'netlify/site/deploy-preview']}); + assert.deepEqual(refs, ['tested-head']); + assert.equal(outputs.sha, 'tested-head'); + assert.equal(outputs['base-url'], 'https://deploy-preview-7--app.netlify.app'); + assert.equal(Object.keys(JSON.parse(outputs['deploy-urls'])).length, 2); +}); + +test('failed, absent and pending previews cannot become passing URL tests', async () => { + for (const state of ['failure', 'error']) { + await assert.rejects(scenario({statuses: [{context: 'netlify/app/deploy-preview', state}]}), /failed/); + } + for (const statuses of [[], [{context: 'netlify/app/deploy-preview', state: 'pending'}]]) { + await assert.rejects(scenario({statuses}), /within 12 minutes/); + } +}); + +test('superseded, closed and draft PRs stop before testing', async () => { + for (const current of [{head: {sha: 'new-head'}}, {state: 'closed'}, {draft: true}]) { + await assert.rejects(scenario({current}), /PR changed or returned to draft/); + } +}); + +test('production, another PR and untrusted URLs are rejected', async () => { + for (const target_url of ['https://app.netlify.app', 'https://deploy-preview-8--app.netlify.app', + 'https://deploy-preview-7--another-site.netlify.app', + 'http://deploy-preview-7--app.netlify.app', 'https://deploy-preview-7--app.netlify.app.attacker.test', + 'https://user:password@deploy-preview-7--app.netlify.app']) { + await assert.rejects(scenario({statuses: [{context: 'netlify/app/deploy-preview', state: 'success', target_url}]}), + /did not return this PR/); + } + await assert.rejects(scenario({names: []}), /require preview-checks/); +}); diff --git a/tests/ready.test.mjs b/tests/ready.test.mjs index d698800..bfb35bb 100644 --- a/tests/ready.test.mjs +++ b/tests/ready.test.mjs @@ -4,7 +4,7 @@ import test from 'node:test'; const source = readFileSync(new URL('../.github/workflows/ready.yml', import.meta.url), 'utf8') .split(' script: |\n')[1].split('\n').map(line => line.slice(12)).join('\n'); -const execute = new (Object.getPrototypeOf(async function () {}).constructor)('github', 'context', 'core', 'process', 'fetch', source); +const execute = new (Object.getPrototypeOf(async function () {}).constructor)('github', 'context', 'core', 'process', 'fetch', 'setTimeout', source); async function scenario(options = {}) { const pull = {number: 1, node_id: 'PR1', state: 'open', draft: false, head: {sha: 'head', ref: 'preview', repo: {full_name: 'o/r'}}, @@ -15,6 +15,14 @@ async function scenario(options = {}) { const statuses = []; const merges = []; let reads = 0; + let runReads = 0; + let jobReads = 0; + const checks = () => (options.checks ?? [{id: 1, name: 'Check / Check', app: {id: 15368}, conclusion: 'success'}]).map(check => + Object.hasOwn(check, 'details_url') ? check : {...check, details_url: `https://github.com/o/r/actions/runs/${check.id}/job/${check.id}`}); + const defaultRuns = () => checks().filter(check => check.name === 'Test / Test' && check.app?.id === 15368) + .map(check => ({id: Number(check.details_url?.match(/\/actions\/runs\/(\d+)/)?.[1]), run_attempt: 1, + name: 'Test', path: '.github/workflows/test.yml', head_sha: 'head', status: check.status, conclusion: check.conclusion})) + .filter(run => run.id); const github = { rest: { pulls: {get: async ({pull_number}) => ({data: ++reads >= 2 && options.changed ? {...pull, head: {sha: 'new'}} : pulls.find(candidate => candidate.number === pull_number)}), @@ -27,10 +35,13 @@ async function scenario(options = {}) { }, createCommitStatus: async status => statuses.push(status)}, checks: {listForRef: 'checks'}, + actions: {listWorkflowRunsForRepo: 'runs', listJobsForWorkflowRunAttempt: 'jobs'}, }, - paginate: async endpoint => endpoint === 'reviews' ? (options.reviewRead ? options.reviewRead() : options.reviews ?? []) - : endpoint === 'pulls' ? pulls : (options.checks ?? [{id: 1, name: 'Check / Check', app: {id: 15368}, conclusion: 'success'}]).map(check => - Object.hasOwn(check, 'details_url') ? check : {...check, details_url: `https://github.com/o/r/actions/runs/${check.id}/job/${check.id}`}), + paginate: async (endpoint, parameters) => endpoint === 'reviews' ? (options.reviewRead ? options.reviewRead() : options.reviews ?? []) + : endpoint === 'pulls' ? pulls + : endpoint === 'runs' ? options.runRead ? options.runRead(++runReads) : options.runs ?? defaultRuns() + : endpoint === 'jobs' ? options.jobRead ? options.jobRead(++jobReads, parameters) : options.jobs ?? checks() + : checks(), request: async () => ({data: [{type: 'required_status_checks', parameters: { required_status_checks: (options.required ?? ['Check / Check', 'Review']).map(context => ({context})), }}]}), @@ -45,8 +56,9 @@ async function scenario(options = {}) { }; await execute(github, {repo: {owner: 'o', repo: 'r'}, payload: options.event ?? {pull_request: {number: 1}}}, {info: () => {}, error: message => errors.push(message), setFailed: message => errors.push(message)}, {env: {CHECK_NAME: 'Check / Check', NETLIFY_CHECKS: JSON.stringify(options.netlify ?? []), + REQUIRE_CODERABBIT: String(options.requireCoderabbit ?? true), TEST_CHECKS: JSON.stringify(options.testChecks ?? []), GITHUB_API_URL: 'https://api.github.com', MERGE_TOKEN: options.mergeToken}}, - options.fetch ?? (() => { throw new Error('Unexpected network request'); })); + options.fetch ?? (() => { throw new Error('Unexpected network request'); }), resolve => resolve()); return {statuses, merges, errors}; } @@ -258,3 +270,108 @@ test('ordinary Ready code PRs pass Review without any automatic merge', async () } } }); + +test('trial repositories no longer depend on CodeRabbit selection or quota', async () => { + const result = await scenario({selected: true, requireCoderabbit: false, changedNotifier: true, + pull: {user: {type: 'User', login: 'silveltman'}}}); + assert.equal(result.statuses[0].state, 'success'); + assert.equal(result.merges.length, 0); +}); + +test('required tests gate Review for ordinary PRs without adding an organization rule', async () => { + const options = {requireCoderabbit: false, testChecks: ['Test / Test'], + pull: {user: {type: 'User', login: 'silveltman'}}}; + for (const check of [null, + {id: 1, name: 'Test / Test', app: {id: 15368}, status: 'completed', conclusion: 'skipped'}, + {id: 1, name: 'Test / Test', app: {id: 15368}, status: 'completed', conclusion: 'failure'}, + {id: 1, name: 'Test / Test', app: {id: 15368}, status: 'in_progress', conclusion: null}, + {id: 1, name: 'Test / Test', app: {id: 1}, status: 'completed', conclusion: 'success'}]) { + assert.equal((await scenario({...options, checks: check ? [check] : []})).statuses.at(-1).state, 'pending'); + } + const passed = {id: 2, name: 'Test / Test', app: {id: 15368}, status: 'completed', conclusion: 'success'}; + assert.equal((await scenario({...options, checks: [passed]})).statuses.at(-1).state, 'success'); + assert.equal((await scenario({...options, checks: [passed, {...passed, id: 3, status: 'in_progress', conclusion: null}]})).statuses.at(-1).state, 'pending'); + assert.equal((await scenario({...options, checks: [{...passed, details_url: null}]})).statuses.at(-1).state, 'pending'); + assert.equal((await scenario({...options, checks: [passed], event: {workflow_run: { + name: 'Test', head_sha: 'head', id: 3, status: 'queued'}}})).statuses.at(-1).state, 'pending'); + assert.equal((await scenario({...options, checks: [passed], event: {workflow_run: { + name: 'Test', head_sha: 'head', id: 3, status: 'completed', conclusion: 'failure'}}})).statuses.at(-1).state, 'pending'); + assert.equal((await scenario({...options, checks: [passed], runs: [{id: 2, run_attempt: 1, name: 'Test', path: '.github/workflows/test.yml', head_sha: 'head', status: 'completed', conclusion: 'failure'}], event: {workflow_run: { + name: 'Test', head_sha: 'head', id: 2, status: 'completed', conclusion: 'failure'}}})).statuses.at(-1).state, 'pending'); +}); + +const trial = {requireCoderabbit: false, testChecks: ['Test / Test'], pull: {user: {type: 'User', login: 'silveltman'}}}; +const testRun = {id: 100, run_attempt: 1, name: 'Test', path: '.github/workflows/test.yml', head_sha: 'head', status: 'completed', conclusion: 'success'}; +const testJob = {id: 200, name: 'Test / Test', status: 'completed', conclusion: 'success'}; + +test('Check events cannot pass a Test rerun using the preceding successful check', async () => { + const result = await scenario({...trial, checks: [{...testJob, app: {id: 15368}}], + runs: [{...testRun, run_attempt: 2, status: 'in_progress', conclusion: null}], jobs: [testJob], + event: {workflow_run: {name: 'Check', head_sha: 'head', status: 'completed', conclusion: 'success'}}}); + assert.equal(result.statuses.at(-1).state, 'pending'); +}); + +test('the exact latest Test attempt supplies every required successful job', async () => { + for (const jobs of [[], [{...testJob, conclusion: 'skipped'}], [{...testJob, conclusion: 'failure'}]]) { + const result = await scenario({...trial, runs: [{...testRun, run_attempt: 2}], jobRead: (_, parameters) => { + assert.equal(parameters.run_id, 100); + assert.equal(parameters.attempt_number, 2); + return jobs; + }}); + assert.equal(result.statuses.at(-1).state, 'pending'); + } + const result = await scenario({...trial, runs: [{...testRun, run_attempt: 2}], jobs: [testJob]}); + assert.equal(result.statuses.at(-1).state, 'success'); +}); + +test('completion retries metadata lag and passes without requiring another event', async () => { + let jobReads = 0; + const result = await scenario({...trial, + event: {workflow_run: {...testRun, id: 101}}, + runRead: reads => [reads === 1 ? testRun : {...testRun, id: 101}], + jobRead: () => ++jobReads === 1 ? [] : [testJob]}); + assert.equal(result.statuses.at(-1).state, 'success'); + assert.ok(jobReads >= 2); +}); + +test('persistent missing metadata is bounded and API errors revoke old success', async () => { + let reads = 0; + const missing = await scenario({...trial, runRead: () => {reads++; return [];}}); + assert.equal(reads, 4); + assert.equal(missing.statuses.at(-1).state, 'pending'); + const failed = await scenario({...trial, runRead: () => {throw new Error('Metadata unavailable');}}); + assert.equal(failed.statuses.at(-1).state, 'pending'); + assert.match(failed.errors[0], /Metadata unavailable/); +}); + +test('completion retries the same run and attempt while its status is stale', async () => { + let runReads = 0; + let jobReads = 0; + const result = await scenario({...trial, event: {workflow_run: testRun}, + runRead: () => [++runReads === 1 ? {...testRun, status: 'in_progress', conclusion: null} : testRun], + jobRead: () => {jobReads++; return [testJob];}}); + assert.equal(result.statuses.at(-1).state, 'success'); + assert.ok(runReads >= 3); + assert.equal(jobReads, 2); + assert.deepEqual(result.errors, []); +}); + +test('a completed older attempt cannot authorize a newer pending attempt', async () => { + const result = await scenario({...trial, event: {workflow_run: testRun}, + runs: [{...testRun, run_attempt: 2, status: 'in_progress', conclusion: null}], jobs: [testJob]}); + assert.equal(result.statuses.at(-1).state, 'pending'); + assert.deepEqual(result.errors, []); +}); + +test('persistent lag after successful completion fails Ready instead of silently waiting', async () => { + for (const options of [ + {runs: [{...testRun, status: 'in_progress', conclusion: null}]}, + {runs: [testRun], jobs: []}, + {runs: []}, + ]) { + const result = await scenario({...trial, ...options, event: {workflow_run: testRun}}); + assert.equal(result.statuses.at(-1).state, 'pending'); + assert.match(result.errors[0], /metadata is still incomplete/); + assert.match(result.errors.at(-1), /could not be reconciled/); + } +});