diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 8404dc2..2074b18 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -9,6 +9,10 @@ on: command: type: string default: vp run check:source + test-command: + description: Optional unit and service tests, using the same installation after checks. + type: string + default: "" install: type: boolean default: true @@ -48,3 +52,8 @@ jobs: env: CHECK_COMMAND: ${{ inputs.command }} run: bash -e -o pipefail -c "$CHECK_COMMAND" + - name: Unit and service tests + if: ${{ inputs.test-command != '' }} + env: + TEST_COMMAND: ${{ inputs.test-command }} + run: bash -e -o pipefail -c "$TEST_COMMAND" diff --git a/.github/workflows/test-live.yml b/.github/workflows/test-live.yml index 3a0ce9b..9f066b6 100644 --- a/.github/workflows/test-live.yml +++ b/.github/workflows/test-live.yml @@ -10,9 +10,13 @@ on: type: string required: true revision: - description: Commit confirmed as published before dispatching this workflow. + description: Full commit SHA to test after deployment; requires prior confirmation if deployments is empty. type: string required: true + deployments: + description: Optional JSON array of production origins and their Netlify config/base paths. Waits for this revision or unchanged build inputs. + type: string + default: "[]" http: type: boolean default: true @@ -27,7 +31,7 @@ jobs: live: name: Live runs-on: ${{ inputs.runner }} - timeout-minutes: 15 + timeout-minutes: 30 env: pnpm_config_ignore_scripts: "true" pnpm_config_ignore_pnpmfile: "true" @@ -37,6 +41,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ inputs.revision }} + fetch-depth: 0 persist-credentials: false - uses: voidzero-dev/setup-vp@68f39c78aaf39be468cabd2afb09c162574c84af # v1.21.2 with: @@ -47,12 +52,82 @@ jobs: args: - --frozen-lockfile - --ignore-scripts - - name: Safe live HTTP smoke - if: ${{ inputs.http }} - run: vp run test:http - name: Install desktop browsers if: ${{ inputs.browser }} run: vp exec playwright install --with-deps chromium firefox webkit + - name: Wait for the published production revision + if: ${{ inputs.deployments != '[]' }} + env: + DEPLOYMENTS: ${{ inputs.deployments }} + REVISION: ${{ inputs.revision }} + shell: python + run: | + import concurrent.futures + import json + import os + import re + import subprocess + import time + import tomllib + import urllib.request + + revision = os.environ["REVISION"] + sites = json.loads(os.environ["DEPLOYMENTS"]) + if not re.fullmatch(r"[0-9a-f]{40}", revision) or not isinstance(sites, list) or not sites: + raise ValueError("Production tests require a full revision and deployment targets.") + for site in sites: + if not re.fullmatch(r"https://[a-z0-9.-]+", site["url"]): + raise ValueError("Production targets must be HTTPS origins.") + + def published(site): + try: + request = urllib.request.Request( + site["url"] + "/.well-known/build.json?revision=" + revision, + headers={"Cache-Control": "no-cache"}, + ) + with urllib.request.urlopen(request, timeout=10) as response: + deployed = json.load(response)["revision"] + except (OSError, ValueError, KeyError, TypeError): + return False + if not isinstance(deployed, str) or not re.fullmatch(r"[0-9a-f]{40}", deployed): + return False + if deployed == revision: + return True + ancestor = subprocess.run( + ["git", "merge-base", "--is-ancestor", deployed, revision], + capture_output=True, + ) + if ancestor.returncode: + return False + # Reuse Netlify's own decision, including its working directory. + # An older deploy is valid only when these build inputs did not change. + with open(site.get("config", "netlify.toml"), "rb") as config: + ignore = tomllib.load(config).get("build", {}).get("ignore") + if not ignore: + return False + result = subprocess.run( + ["bash", "-e", "-o", "pipefail", "-c", ignore], + cwd=site.get("base", "."), + env={**os.environ, "CACHED_COMMIT_REF": deployed, "COMMIT_REF": revision}, + timeout=10, + ) + if result.returncode not in (0, 1): + raise RuntimeError("Netlify's build ignore command failed for " + site["url"]) + return result.returncode == 0 + + deadline = time.monotonic() + 12 * 60 + pending = sites + with concurrent.futures.ThreadPoolExecutor(max_workers=12) as pool: + while pending and time.monotonic() < deadline: + pending = [site for site, ready in zip(pending, pool.map(published, pending)) if not ready] + if pending: + print("Waiting for production: " + ", ".join(site["url"] for site in pending), flush=True) + time.sleep(15) + if pending: + raise RuntimeError("Matching production deploys did not complete within 12 minutes.") + - name: Safe live HTTP smoke + if: ${{ inputs.http }} + run: vp run test:http - name: Safe live browser smoke if: ${{ inputs.browser }} # A virtual display on Linux, for repositories that run a browser headed. diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3ae78c0..7c758c4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -78,6 +78,23 @@ jobs: pnpm_config_ignore_pnpmfile: "true" QA_PROFILE: preview steps: + - name: Checkout the tested preview revision + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - uses: voidzero-dev/setup-vp@68f39c78aaf39be468cabd2afb09c162574c84af # v1.21.2 + with: + version: "1.1.0" + node-version-file: ${{ inputs.node-version-file }} + cache: true + run-install: | + args: + - --frozen-lockfile + - --ignore-scripts + - name: Install desktop browsers + if: ${{ inputs.browser }} + run: vp exec playwright install --with-deps chromium firefox webkit - name: Resolve previews for this PR revision id: preview uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 @@ -126,29 +143,12 @@ jobs: await new Promise(resolve => setTimeout(resolve, 15000)); } throw new Error('Matching previews did not complete within 12 minutes.'); - - name: Checkout the tested preview revision - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ steps.preview.outputs.sha }} - persist-credentials: false - - uses: voidzero-dev/setup-vp@68f39c78aaf39be468cabd2afb09c162574c84af # v1.21.2 - with: - version: "1.1.0" - node-version-file: ${{ inputs.node-version-file }} - cache: true - run-install: | - args: - - --frozen-lockfile - - --ignore-scripts - name: HTTP tests if: ${{ inputs.http }} env: BASE_URL: ${{ steps.preview.outputs.base-url }} QA_DEPLOY_URLS: ${{ steps.preview.outputs.deploy-urls }} run: vp run test:http - - name: Install desktop browsers - if: ${{ inputs.browser }} - run: vp exec playwright install --with-deps chromium firefox webkit - name: Browser tests id: browser if: ${{ inputs.browser }} diff --git a/README.md b/README.md index 24adf79..cec4dd3 100644 --- a/README.md +++ b/README.md @@ -6,17 +6,23 @@ Callers pin a full commit SHA from `main` after the shared workflow PR is merged New projects take their callers from [astro-template](https://github.com/fulldotdev/astro-template) or [vite-template](https://github.com/fulldotdev/vite-template). -- **Develop** stays local while implementing and experimenting. -- **Draft PR** builds and checks artifacts locally, then opens a shared draft and Netlify preview. +- **Explore** experiments locally; **Implement** builds the requested change locally. +- **Draft PR** builds locally, then opens a shared draft and a Netlify preview. Source checks and tests do not delay the preview. - **Make PR ready** runs enabled checks and tests locally, then CI confirms them. Source checks and unit tests run alongside the Netlify build. URL tests use the preview of the tested PR head. - **Merge PR** merges the validated head and follows the production build and short, safe live checks. -Node repositories expose `check:source` and the categories they actually use. Existing unit tests run through `test:unit`; artifact checks consume the build through `check:build`. Enable HTTP and browser tests only where the repository has useful tests and a matching deployment target. Devices stay manual. Do not add empty scripts or test workflows for disabled categories. +Fulldev, astro-template and vite-template expose `fix`, `check`, `build` and `test`. `check` validates source; `build` includes intrinsic artifact validation. Internal `test:unit`, `test:http` and `test:browser` helpers let CI run the enabled suites in the right environment. Devices stay manual. Do not add empty scripts or test workflows for disabled categories. + +These callers use the Check workflow's optional `test-command` to run unit and service tests after source checks with one installation. Their Test caller disables its separate unit job. Browser dependencies install before waiting for the exact PR revision's previews. + +After a push to main, Check runs source checks and unit/service tests again, Netlify runs the same build command, and Test live waits for the published revision before safe live HTTP/browser tests. Its `deployments` input lists HTTPS origins and their repository-relative Netlify config/base paths. Each build publishes `/.well-known/build.json` with its `COMMIT_REF`. For an older production deploy, Test live accepts it only if it is an ancestor of the requested revision and that site's existing Netlify `build.ignore` command confirms unchanged inputs. No Netlify token or second build is needed. + +Other repositories retain their existing `check:source`, `check:build` and Test caller contracts until explicitly migrated. The reusable defaults remain compatible with them. Non-Node repositories keep their existing language-specific check command with dependency installation disabled. Repositories with `test.yml` require `Test / Test` through the existing `Review` status. Missing, skipped, failed or superseded test results cannot pass that gate. Independent agent review owns code assessment; Ready has no external review-service dependency. -The Ready caller grants `actions: read` so the gate reads the current Test workflow attempt and its jobs, including reruns. Non-Node repositories keep their existing language-specific check command with dependency installation disabled. +The Ready caller grants `actions: read` so the gate reads the current Test workflow attempt and its jobs, including reruns. -Organization rules require `Check / Check` and `Review`, without requiring synchronization with main. Netlify requirements are managed centrally for repositories with those sites. Production keeps scoped source, build and artifact checks. Live URL tests run after the current production deployment has been confirmed, rather than inside its build command. +Organization rules require `Check / Check` and `Review`, without requiring synchronization with main. Netlify requirements are managed centrally for repositories with those sites. Live URL tests run after deployment, outside the build command; signed-in tests and writes stay on previews. Keep repository auto-merge disabled except for CloudCannon publishers. CloudCannon's Publish button keeps automatic publication for same-repository `preview` PRs created by `cloudcannon[bot]`, with all required checks and previews. Code PRs are merged by their agent. diff --git a/tests/preview.test.mjs b/tests/preview.test.mjs index 66c4b7d..e47e400 100644 --- a/tests/preview.test.mjs +++ b/tests/preview.test.mjs @@ -3,7 +3,7 @@ import {readFileSync} from 'node:fs'; import test from 'node:test'; const source = readFileSync(new URL('../.github/workflows/test.yml', import.meta.url), 'utf8') - .split(' script: |\n')[1].split('\n - name: Checkout')[0] + .split(' script: |\n')[1].split('\n - name: HTTP tests')[0] .split('\n').map(line => line.slice(12)).join('\n'); const execute = new (Object.getPrototypeOf(async function () {}).constructor)( 'github', 'context', 'core', 'process', 'Date', 'setTimeout', source); diff --git a/tests/production.test.mjs b/tests/production.test.mjs new file mode 100644 index 0000000..8af8664 --- /dev/null +++ b/tests/production.test.mjs @@ -0,0 +1,63 @@ +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import {spawnSync} from 'node:child_process'; +import test from 'node:test'; + +const source = readFileSync(new URL('../.github/workflows/test-live.yml', import.meta.url), 'utf8') + .split(' run: |\n')[1].split('\n - name: Safe live HTTP smoke')[0] + .split('\n').map(line => line.slice(10)).join('\n').split('deadline =')[0]; + +test('production waits for this revision and reuses the Netlify ignore decision for unchanged sites', () => { + const result = spawnSync('python3', ['-c', ` +import contextlib, io, json, os, tempfile +from types import SimpleNamespace +from unittest.mock import patch + +os.environ['REVISION'] = 'a' * 40 +os.environ['DEPLOYMENTS'] = '[{"url":"https://example.netlify.app"}]' +scope = {} +exec(${JSON.stringify(source)}, scope) +published = scope['published'] +site = {'url': 'https://example.netlify.app'} + +def response(revision): + return contextlib.nullcontext(io.StringIO(json.dumps({'revision': revision}))) + +with patch('urllib.request.urlopen', return_value=response('a' * 40)), patch('subprocess.run') as run: + assert published(site) + run.assert_not_called() + +for revision in [None, 'bad', '$(unsafe)', 'a' * 41]: + with patch('urllib.request.urlopen', return_value=response(revision)), patch('subprocess.run') as run: + assert not published(site) + run.assert_not_called() + +with patch('urllib.request.urlopen', side_effect=OSError('not deployed')): + assert not published(site) + +with tempfile.TemporaryDirectory() as directory: + config = os.path.join(directory, 'netlify.toml') + with open(config, 'w') as handle: + handle.write("[build]\\nignore = 'git diff --quiet'\\n") + target = {**site, 'config': config, 'base': directory} + for code in [0, 1, 2]: + with patch('urllib.request.urlopen', return_value=response('b' * 40)), patch('subprocess.run', + side_effect=[SimpleNamespace(returncode=0), SimpleNamespace(returncode=code)]) as run: + if code == 2: + try: + published(target) + raise AssertionError('ignore errors cannot pass') + except RuntimeError: + pass + else: + assert published(target) == (code == 0) + assert run.call_args.kwargs['cwd'] == directory + assert run.call_args.kwargs['env']['CACHED_COMMIT_REF'] == 'b' * 40 + assert run.call_args.kwargs['env']['COMMIT_REF'] == 'a' * 40 + with patch('urllib.request.urlopen', return_value=response('c' * 40)), patch('subprocess.run', + return_value=SimpleNamespace(returncode=1)) as run: + assert not published(target) + assert run.call_count == 1 +`], {encoding: 'utf8'}); + assert.equal(result.status, 0, result.stderr || result.stdout); +});