From f0d09fe1666aa1412be6354959050cfdfd2351bf Mon Sep 17 00:00:00 2001 From: woksin Date: Fri, 23 Oct 2026 02:49:05 +0200 Subject: [PATCH 1/2] Make publish template placeholders fail loudly --- .github/templates/publish.template.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/templates/publish.template.yml b/.github/templates/publish.template.yml index 19b37e1..4b60d1d 100644 --- a/.github/templates/publish.template.yml +++ b/.github/templates/publish.template.yml @@ -87,6 +87,8 @@ jobs: steps: # ... publish dotnet packages ... + - name: Publish .NET packages + run: echo "::error::Replace this step with your .NET publish steps" && exit 1 publish-npm-packages: if: needs.release.outputs.publish == 'true' @@ -98,4 +100,6 @@ jobs: steps: # ... publish npm packages ... + - name: Publish npm packages + run: echo "::error::Replace this step with your npm publish steps" && exit 1 From f52361ba6af1eb3812966c3f29048c14ee4e2b97 Mon Sep 17 00:00:00 2001 From: woksin Date: Fri, 23 Oct 2026 02:49:21 +0200 Subject: [PATCH 2/2] Add reusable workflow lint and PR smoke coverage --- .github/actionlint.yaml | 13 ++ .github/scripts/install-actionlint.sh | 27 ++++ .../broken/.github/workflows/broken.yml | 19 +++ .../good/.github/workflows/queue.yml | 15 +++ .../fixtures/workflow-lint/no-ignores.yaml | 3 + .github/scripts/tests/workflow-lint.test.py | 118 ++++++++++++++++++ .github/workflows/smoke-workflows.yml | 60 +++++++++ .../verify-release-notes-program.yml | 12 +- .github/workflows/workflow-lint.yml | 67 ++++++++++ README.md | 73 +++++++++++ 10 files changed, 396 insertions(+), 11 deletions(-) create mode 100644 .github/actionlint.yaml create mode 100644 .github/scripts/install-actionlint.sh create mode 100644 .github/scripts/tests/fixtures/workflow-lint/broken/.github/workflows/broken.yml create mode 100644 .github/scripts/tests/fixtures/workflow-lint/good/.github/workflows/queue.yml create mode 100644 .github/scripts/tests/fixtures/workflow-lint/no-ignores.yaml create mode 100644 .github/scripts/tests/workflow-lint.test.py create mode 100644 .github/workflows/smoke-workflows.yml create mode 100644 .github/workflows/workflow-lint.yml diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 0000000..839179d --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,13 @@ +# Copyright (c) Cratis. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. +self-hosted-runner: + labels: + - cratis-arc + - cratis-arc-small + - cratis-arc-release + +paths: + '.github/{workflows,templates}/**': + ignore: + # Remove once actionlint supports concurrency.queue. + - 'unexpected key "queue" for "concurrency" section' diff --git a/.github/scripts/install-actionlint.sh b/.github/scripts/install-actionlint.sh new file mode 100644 index 0000000..b284689 --- /dev/null +++ b/.github/scripts/install-actionlint.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Copyright (c) Cratis. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. +set -euo pipefail + +version=1.7.12 +case "$(uname -m)" in + x86_64|amd64) + arch=amd64 + sha256=8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8 + ;; + aarch64|arm64) + arch=arm64 + sha256=325e971b6ba9bfa504672e29be93c24981eeb1c07576d730e9f7c8805afff0c6 + ;; + *) + echo "Unsupported actionlint architecture: $(uname -m)" >&2 + exit 1 + ;; +esac + +archive="$RUNNER_TEMP/actionlint.tar.gz" +curl -fsSL "https://github.com/rhysd/actionlint/releases/download/v${version}/actionlint_${version}_linux_${arch}.tar.gz" -o "$archive" +echo "$sha256 $archive" | sha256sum --check --strict +mkdir -p "$RUNNER_TEMP/bin" +tar -xzf "$archive" -C "$RUNNER_TEMP/bin" actionlint +echo "$RUNNER_TEMP/bin" >> "$GITHUB_PATH" diff --git a/.github/scripts/tests/fixtures/workflow-lint/broken/.github/workflows/broken.yml b/.github/scripts/tests/fixtures/workflow-lint/broken/.github/workflows/broken.yml new file mode 100644 index 0000000..afaaf99 --- /dev/null +++ b/.github/scripts/tests/fixtures/workflow-lint/broken/.github/workflows/broken.yml @@ -0,0 +1,19 @@ +# Copyright (c) Cratis. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. +name: Broken fixture +on: + workflow_call: + inputs: + declared: + type: string +concurrency: + group: fixture + invalid-key: true +jobs: + check: + needs: missing-job + runs-on: ubuntu-latest + steps: + - run: echo fixture + env: + UNDEFINED: ${{ inputs.nope }} diff --git a/.github/scripts/tests/fixtures/workflow-lint/good/.github/workflows/queue.yml b/.github/scripts/tests/fixtures/workflow-lint/good/.github/workflows/queue.yml new file mode 100644 index 0000000..c17f353 --- /dev/null +++ b/.github/scripts/tests/fixtures/workflow-lint/good/.github/workflows/queue.yml @@ -0,0 +1,15 @@ +# Copyright (c) Cratis. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. +name: Shared configuration fixture +on: workflow_dispatch +concurrency: + group: fixture + queue: max +jobs: + check: + strategy: + matrix: + runner: [cratis-arc, cratis-arc-small, cratis-arc-release] + runs-on: ${{ matrix.runner }} + steps: + - run: echo fixture diff --git a/.github/scripts/tests/fixtures/workflow-lint/no-ignores.yaml b/.github/scripts/tests/fixtures/workflow-lint/no-ignores.yaml new file mode 100644 index 0000000..e848e6f --- /dev/null +++ b/.github/scripts/tests/fixtures/workflow-lint/no-ignores.yaml @@ -0,0 +1,3 @@ +# Copyright (c) Cratis. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. +{} diff --git a/.github/scripts/tests/workflow-lint.test.py b/.github/scripts/tests/workflow-lint.test.py new file mode 100644 index 0000000..768e1f3 --- /dev/null +++ b/.github/scripts/tests/workflow-lint.test.py @@ -0,0 +1,118 @@ +# Copyright (c) Cratis. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. +"""Offline actionlint red controls and reusable-workflow smoke coverage.""" +import os +from pathlib import Path +import re +import shutil +import subprocess +import tempfile +import unittest + +import yaml + +ROOT = Path(__file__).resolve().parents[3] +WORKFLOWS = ROOT / ".github/workflows" +CONFIG = ROOT / ".github/actionlint.yaml" +FIXTURES = Path(__file__).parent / "fixtures/workflow-lint" +NOT_SMOKEABLE = { + "update-packages.yml": "requires secrets and writes package updates", + "cleanup-pr-artifacts.yml": "requires secrets and deletes repository artifacts", + "trigger-documentation-build.yml": "dispatches to another repository using a secret", + "normalize-dependabot-labels.yml": "writes pull request labels", + "verify-pull-request-policy.yml": "holds Workflows PRs to consumer release policy; decide separately", + "verify-release-notes.yml": "holds Workflows PRs to consumer release policy; decide separately", + "verify-release-intent.yml": "holds Workflows PRs to consumer release policy; decide separately", +} + + +def document(path): + return yaml.safe_load(path.read_text()) + + +def triggers(workflow): + # PyYAML's YAML 1.1 loader reads the unquoted `on` key as True. + value = workflow.get("on", workflow.get(True, {})) + return {value} if isinstance(value, str) else set(value) + + +def lint(case, config=True): + args = ["actionlint", "-pyflakes=", "-shellcheck="] + # Always specify a config: discovery would otherwise find the repository's shared config. + args += ["-config-file", str(CONFIG if config else FIXTURES / "no-ignores.yaml")] + files = sorted((FIXTURES / case / ".github/workflows").glob("*.yml")) + return subprocess.run(args + [str(path) for path in files], cwd=FIXTURES / case, + capture_output=True, text=True, timeout=30) + + +class WorkflowLintTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + if not shutil.which("actionlint"): + raise RuntimeError("actionlint is required; run .github/scripts/install-actionlint.sh in CI") + + def test_broken_fixture_fails_with_expected_rule_ids(self): + result = lint("broken") + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertEqual(set(re.findall(r"\[([a-z-]+)\]", result.stdout)), + {"expression", "job-needs", "syntax-check"}, result.stdout) + for diagnostic in ['property "nope"', '"missing-job"', 'unexpected key "invalid-key"']: + self.assertIn(diagnostic, result.stdout) + + def test_queue_and_shared_runner_labels_pass_with_shared_config(self): + result = lint("good") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + # A red control proves the fixture actually needs the narrowly scoped exception. + without_config = lint("good", config=False) + self.assertEqual(without_config.returncode, 1, without_config.stdout + without_config.stderr) + self.assertIn('unexpected key "queue" for "concurrency" section', without_config.stdout) + + def test_every_reusable_is_smoked_or_has_an_explicit_reason(self): + reusables = {path.name for path in WORKFLOWS.iterdir() + if path.suffix in {".yml", ".yaml"} and "workflow_call" in triggers(document(path))} + smoke = document(WORKFLOWS / "smoke-workflows.yml") + called = {job["uses"].removeprefix("./.github/workflows/") + for job in smoke["jobs"].values() if "uses" in job} + self.assertTrue(reusables, "No reusable workflows were discovered") + self.assertTrue(called, "No reusable workflows are smoke-called") + self.assertEqual(called & NOT_SMOKEABLE.keys(), set(), "Remove obsolete exemptions") + self.assertEqual(reusables, called | NOT_SMOKEABLE.keys(), + "Smoke-call new reusables or give a concrete NOT_SMOKEABLE reason") + self.assertTrue(all(reason.strip() for reason in NOT_SMOKEABLE.values())) + self.assertEqual(triggers(smoke), {"pull_request"}) + self.assertEqual(smoke["permissions"], {"contents": "read"}) + for job in smoke["jobs"].values(): + if "uses" in job: + self.assertTrue(job["uses"].startswith("./.github/workflows/")) + self.assertNotIn("secrets", job) + + def test_lint_step_uses_repo_config_or_shared_default_and_keeps_workflows_with_extras(self): + workflow = document(WORKFLOWS / "workflow-lint.yml") + script = workflow["jobs"]["lint"]["steps"][-1]["run"] + for config_name in ["actionlint.yaml", "actionlint.yml", None]: + with self.subTest(config=config_name), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + shutil.copytree(FIXTURES / "good/.github", root / ".github") + shared = root / ".cratis-workflows/.github" + shared.mkdir(parents=True) + shutil.copy(CONFIG, shared / "actionlint.yaml") + if config_name: + shutil.copy(CONFIG, root / ".github" / config_name) + (shared / "actionlint.yaml").write_text("invalid config: true\n") + extra = root / ".github/templates/extra.yml" + extra.parent.mkdir() + shutil.copy(FIXTURES / "good/.github/workflows/queue.yml", extra) + env = dict(os.environ, SHELLCHECK="false", EXTRA_FILES=str(extra.relative_to(root))) + result = subprocess.run(["bash", "-c", script], cwd=root, env=env, + capture_output=True, text=True, timeout=30) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + # A valid extra file must not hide a broken workflow from discovery. + shutil.copy(FIXTURES / "broken/.github/workflows/broken.yml", root / ".github/workflows") + result = subprocess.run(["bash", "-c", script], cwd=root, env=env, + capture_output=True, text=True, timeout=30) + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertIn("[job-needs]", result.stdout) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/.github/workflows/smoke-workflows.yml b/.github/workflows/smoke-workflows.yml new file mode 100644 index 0000000..d1eedf6 --- /dev/null +++ b/.github/workflows/smoke-workflows.yml @@ -0,0 +1,60 @@ +# Copyright (c) Cratis. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. +name: Smoke Workflows + +on: + pull_request: + paths: + - .github/workflows/** + - .github/actions/** + - .github/templates/** + - .github/actionlint.yaml + - .github/scripts/install-actionlint.sh + - .github/scripts/tests/workflow-lint.test.py + - .github/scripts/tests/fixtures/workflow-lint/** + - .github/scripts/tests/fixtures/discoverability/** + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + lint: + uses: ./.github/workflows/workflow-lint.yml + with: + workflows-ref: ${{ github.sha }} + shellcheck: true + extra-files: .github/templates/publish.template.yml + + tests: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Install actionlint + run: bash .github/scripts/install-actionlint.sh + - name: Exercise lint failures and smoke coverage + run: python3 -B .github/scripts/tests/workflow-lint.test.py + + docker-cleanup: + uses: ./.github/workflows/docker-cleanup-gate.yml + + public-runners: + uses: ./.github/workflows/check-public-runners.yml + with: + workflows-ref: ${{ github.sha }} + + no-work-records: + uses: ./.github/workflows/verify-no-work-records.yml + + discoverability: + uses: ./.github/workflows/verify-discoverability.yml + with: + site-path: .github/scripts/tests/fixtures/discoverability/good + base-url: https://example.cratis.io + fail-on: error diff --git a/.github/workflows/verify-release-notes-program.yml b/.github/workflows/verify-release-notes-program.yml index 2bcd3e0..b2a8b9f 100644 --- a/.github/workflows/verify-release-notes-program.yml +++ b/.github/workflows/verify-release-notes-program.yml @@ -63,16 +63,6 @@ jobs: run: node --test .github/scripts/tests/verify-pull-request-policy.spec.mjs - name: Install actionlint # The migration test lints every rendered caller; the runner image has no actionlint. - env: - VERSION: 1.7.12 - SHA256: 8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8 - run: | - set -euo pipefail - archive="$RUNNER_TEMP/actionlint.tar.gz" - curl -fsSL "https://github.com/rhysd/actionlint/releases/download/v${VERSION}/actionlint_${VERSION}_linux_amd64.tar.gz" -o "$archive" - echo "$SHA256 $archive" | sha256sum --check --strict - mkdir -p "$RUNNER_TEMP/bin" - tar -xzf "$archive" -C "$RUNNER_TEMP/bin" actionlint - echo "$RUNNER_TEMP/bin" >> "$GITHUB_PATH" + run: bash .github/scripts/install-actionlint.sh - name: Exercise atomic policy wrapper migration run: python3 -B .github/scripts/tests/bootstrap-policy.test.py diff --git a/.github/workflows/workflow-lint.yml b/.github/workflows/workflow-lint.yml new file mode 100644 index 0000000..4d3550e --- /dev/null +++ b/.github/workflows/workflow-lint.yml @@ -0,0 +1,67 @@ +# Copyright (c) Cratis. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. +name: Workflow Lint + +on: + workflow_call: + inputs: + runs-on: + description: Runner label for the lint job; keep it hosted in a public repository. + type: string + default: ubuntu-latest + workflows-ref: + description: Cratis/Workflows ref providing the installer and default configuration. + type: string + default: main + shellcheck: + description: Also check inline shell scripts with shellcheck. + type: boolean + default: false + extra-files: + description: Space-separated additional workflow files, such as templates, to lint. + type: string + default: "" + +permissions: + contents: read + +jobs: + lint: + runs-on: ${{ inputs.runs-on }} + timeout-minutes: 5 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: Cratis/Workflows + ref: ${{ inputs.workflows-ref }} + path: .cratis-workflows + persist-credentials: false + + - name: Install actionlint + run: bash .cratis-workflows/.github/scripts/install-actionlint.sh + + - name: Lint workflows and additional files + shell: bash + env: + SHELLCHECK: ${{ inputs.shellcheck }} + EXTRA_FILES: ${{ inputs.extra-files }} + run: | + set -euo pipefail + config=.cratis-workflows/.github/actionlint.yaml + for candidate in .github/actionlint.yaml .github/actionlint.yml; do + if [ -f "$candidate" ]; then + config="$candidate" + break + fi + done + args=(-config-file "$config" -pyflakes=) + if [ "$SHELLCHECK" != true ]; then args+=(-shellcheck=); fi + # Explicit file arguments replace discovery, so keep all workflows alongside extras. + shopt -s nullglob + files=(.github/workflows/*.yml .github/workflows/*.yaml) + read -r -a extra_files <<< "$EXTRA_FILES" + actionlint "${args[@]}" "${files[@]}" "${extra_files[@]}" diff --git a/README.md b/README.md index 7702cf7..71a3da9 100644 --- a/README.md +++ b/README.md @@ -720,6 +720,79 @@ The check's program is inline in the reusable workflow, so the rules that run ar --- +## Verifying workflow changes + +`smoke-workflows.yml` runs on `pull_request` changes to workflows, actions, +templates, the actionlint configuration/installer, and its tests and fixtures. +It calls reusable workflows through `./.github/workflows/` so GitHub uses the +PR merge commit, not the version on `main`. The lint and public-runner calls also +pass `${{ github.sha }}` as `workflows-ref` for their supporting files. Permissions +are read-only, no secrets are passed, and public PRs use `ubuntu-latest`. +The path filter is intentional: this workflow is **not a required check**. + +| Coverage | Reusable workflow | Limit | +| --- | --- | --- | +| Smoke call | `workflow-lint.yml` | All workflows plus the publish template; shellcheck enabled | +| Smoke call | `docker-cleanup-gate.yml` | Read-only scan; its checker checkout still defaults to `main` | +| Smoke call | `check-public-runners.yml` | Read-only scan using the PR merge commit's checker | +| Smoke call | `verify-no-work-records.yml` | Uses its existing immutable script pin | +| Smoke call | `verify-discoverability.yml` | Good HTML fixture at `https://example.cratis.io`; existing script pin | +| Not smokeable | `update-packages.yml`, `cleanup-pr-artifacts.yml` | Secrets and repository writes/deletions | +| Not smokeable | `trigger-documentation-build.yml` | Cross-repository dispatch and a secret | +| Not smokeable | `normalize-dependabot-labels.yml` | Writes labels | +| Not smokeable | `verify-pull-request-policy.yml`, `verify-release-notes.yml`, `verify-release-intent.yml` | Holds Workflows PRs to consumer release policy; decide separately | + +`auto-approve-publish-deployments.yml` uses `workflow_run`, not `workflow_call`, +so it is outside the reusable coverage ratchet. Action changes trigger this +workflow, but these smoke calls do not execute composite actions. The offline test +requires every reusable workflow to be called here or have an explicit reason +in `NOT_SMOKEABLE`; adding a reusable without either fails the test. + +### Linting locally and in consumers + +The shared installer pins actionlint 1.7.12 and verifies the Linux amd64/arm64 +archive checksum. `workflow-lint.yml` accepts `runs-on` (default `ubuntu-latest`), +`workflows-ref` (default `main`), `shellcheck` (default `false`), and `extra-files` +(a space-separated list of additional paths). It always lints all caller workflows, +including when extra files are supplied. It prefers the caller's +`.github/actionlint.yaml` or `.github/actionlint.yml`, otherwise uses the shared +configuration. The shared config recognizes Cratis runner labels and ignores +only actionlint's unsupported `concurrency.queue` key under workflows/templates; +remove that exception when actionlint supports it. Pyflakes is disabled. + +```bash +actionlint +actionlint .github/templates/publish.template.yml +python3 -B .github/scripts/tests/workflow-lint.test.py +``` + +The test proves a broken workflow fails with `expression`, `job-needs` and +`syntax-check` rule IDs, and that `queue: max` passes with the shared config. + +### Manual fallback for changes smoke cannot exercise + +For a reusable that can safely be dispatched with its existing permissions, +use a scratch branch in a consumer such as Arc. Point its caller at +`Cratis/Workflows/.github/workflows/@`, ensure that caller +supports `workflow_dispatch`, push the scratch branch, then run: + +```bash +gh workflow run --repo Cratis/Arc --ref +``` + +Dispatch is not a dry run: use only explicitly authorized secrets and effects. +Link the exact run in a **PR comment**, not the release-note description. +Cleanup is an exception: its manual dispatch is main-only and `dry_run` is not a +reusable input. Do not dispatch a scratch cleanup caller (it deletes artifacts). +After the reviewed change lands, use `dry_run: true` in Workflows following +[Retry an already-closed PR safely](#retry-an-already-closed-pr-safely); do not +weaken that main-only boundary to test a branch. + +For consumer `pull_request_target` or publish workflows, use actionlint only +before merge and watch the first authorized run on `main`. Smoke calls do not +prove privileged event semantics, publishing, deployments, or secret-backed +writes, and are not a consumer rollout or a bootstrap installation. + ## Workflows in this repository ### `cleanup-pr-artifacts.yml`