diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9f9fd1f8da..69d62aab51 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,12 +16,19 @@ env: OPENBLAS_NUM_THREADS: "1" MKL_NUM_THREADS: "1" +# GitHub runs an explicit `shell: bash` as `bash -eo pipefail`, and a step +# with no shell as `bash -e`, where a pipe hides the exit status of the command +# that feeds it. +defaults: + run: + shell: bash + # Every job sets timeout-minutes (GitHub's default is 360) so a hung test -# frees its runner from the org's shared 60-runner pool. The engine-free and -# engine-us caps are about 1.5x their slowest successful leg on 2026-10-07..08; -# the engine-uk cap is about twice its slowest successful leg after the -# policyengine-uk 2.162.3 bump (29 minutes, microcosm#1400); shorter jobs get -# at least 5 minutes and at least twice their slowest run. +# frees its runner from the org's shared 60-runner pool. Each cap follows the +# job's slowest successful leg on 2026-11-09, rounded up to a multiple of 5 +# minutes: 1.5 times it for engine-free (109 minutes) and engine-us (73), twice +# for engine-uk (45) and integration-uk (8), and for each shorter job at least +# 5 minutes and at least twice its slowest run. jobs: select-countries: @@ -55,14 +62,14 @@ jobs: run: uv run python tools/graph_acceptance_burndown.py --verify - name: Sync workspace run: uv sync --all-packages --locked + - name: Verify the workflow against the test plan + run: uv run --no-sync python tools/ci_test_plan.py verify-workflow - name: Lint run: uv run --no-sync ruff check . engine-free: runs-on: ubuntu-latest - # Successful legs took 64-74 min on 2026-10-26, and a Python 3.14 leg on - # microcosm#1404 hit the old 75-minute cap; 120 is about 1.6x the slowest. - timeout-minutes: 120 + timeout-minutes: 165 strategy: fail-fast: false matrix: @@ -86,8 +93,7 @@ jobs: needs: select-countries if: needs['select-countries'].outputs.run_us == 'true' runs-on: ubuntu-latest - # Successful legs took 40-60 min on 2026-10-07..08 (n=23). - timeout-minutes: 90 + timeout-minutes: 110 strategy: fail-fast: false matrix: @@ -118,7 +124,7 @@ jobs: needs: select-countries if: needs['select-countries'].outputs.run_uk == 'true' runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 90 strategy: fail-fast: false matrix: @@ -139,7 +145,7 @@ jobs: permissions: contents: read runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 20 steps: - uses: actions/checkout@v4 with: @@ -185,3 +191,34 @@ jobs: run: uv sync --all-packages --locked - name: Run installed-emitter tests run: bash tools/run_installed_emitter_tests.sh + + ci: + # ci gives the main ruleset one check that covers the country jobs. It + # fails unless every other job passed, or skipped because select-countries + # succeeded and reported `false` for its country. A skipped matrix job + # reports under its unexpanded name, so a ruleset cannot require engine-us + # or engine-uk by name. GitHub reports a skipped required check as passing, + # so ci runs with `if: always()`. verify-workflow refuses any other shape + # of this job, and the plan's tests, which engine-free runs, fail on one. + if: always() + needs: + - select-countries + - lint + - engine-free + - engine-us + - engine-uk + - integration-uk + - wheels + - installed-emitter + permissions: + contents: read + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Require every job to pass, or the selection to leave it out + env: + NEEDS: ${{ toJSON(needs) }} + run: python3 tools/ci_test_plan.py require-results diff --git a/changelog.d/ci-aggregate-check.added.md b/changelog.d/ci-aggregate-check.added.md new file mode 100644 index 0000000000..b3957edc97 --- /dev/null +++ b/changelog.d/ci-aggregate-check.added.md @@ -0,0 +1 @@ +Add a `ci` workflow job that gives the `main` ruleset one check covering the country jobs. It fails unless every other job passed, or skipped because `select-countries` succeeded and reported `false` for its country. The lint job runs a new `tools/ci_test_plan.py verify-workflow` command, which parses the workflow and refuses a `ci` job that could skip, omits a job from `needs` or does anything but require the other jobs' results. diff --git a/changelog.d/ci-job-timeouts-2026-11.changed.md b/changelog.d/ci-job-timeouts-2026-11.changed.md new file mode 100644 index 0000000000..fee09774c9 --- /dev/null +++ b/changelog.d/ci-job-timeouts-2026-11.changed.md @@ -0,0 +1 @@ +Raise four job timeouts to at least 1.5 times each job's slowest successful leg on 2026-11-09: engine-free from 120 to 165 minutes (its slowest leg took 109), engine-us from 90 to 110 (73), engine-uk from 60 to 90 (45) and integration-uk from 15 to 20 (8). diff --git a/changelog.d/ci-selection-listing-limit.fixed.md b/changelog.d/ci-selection-listing-limit.fixed.md new file mode 100644 index 0000000000..1835607447 --- /dev/null +++ b/changelog.d/ci-selection-listing-limit.fixed.md @@ -0,0 +1 @@ +Select both countries' engine jobs for a pull request whose file listing reaches GitHub's 3,000-file limit, where the listing stops. diff --git a/docs/agent-guide.md b/docs/agent-guide.md index 40299fa92c..5f17a5c763 100644 --- a/docs/agent-guide.md +++ b/docs/agent-guide.md @@ -42,7 +42,55 @@ or `uv sync --all-packages --all-extras`. To keep both countries set up, point PR CI (`.github/workflows/test.yml`) has `lint`, `engine-free`, `engine-us`, `engine-uk`, `integration-uk`, `wheels`, and `installed-emitter` jobs, plus the -`select-countries` orchestration job. +`select-countries` orchestration job and the `ci` job. + +`ci` gives the `main` ruleset one check that covers the country jobs. It fails +unless every other job passed, or skipped because `select-countries` succeeded +and reported `false` for its country. A skipped matrix job reports under its +unexpanded name, so a ruleset cannot require `engine-us` or `engine-uk` by +name. The ruleset also names each other check that runs on every pull +request: `lint`, `wheels`, `select-countries`, `installed-emitter` and both +`engine-free` legs. When `engine-free`'s matrix changes, change the ruleset +too, because a leg's check name holds its Python version. + +The workflow runs on a pull request only when it targets `main`. Retargeting a +pull request to `main` starts no run, because GitHub reports a base change as +an `edited` event, which the workflow doesn't run on. After a stacked pull +request retargets, push a commit or close and reopen it, then merge once its +checks pass. + +The plan names the jobs the selection may leave out: each job whose test +groups all need the same country's engine. The lint job runs +`tools/ci_test_plan.py verify-workflow`, which parses the workflow with PyYAML +and refuses: + +- two keys that PyYAML reads as one, a merge key (`<<`), and U+0085, U+2028 + or U+2029, which YAML 1.1 reads as line breaks and YAML 1.2 doesn't; +- a job outside the plan, a plan job the workflow lacks, and a job whose `if` + differs from the one the plan gives it; +- a job the selection may leave out that doesn't need `select-countries`, and + a job that needs one the selection may leave out, unless both follow the + same country's selection; +- a `ci` job that lacks `if: always()`, omits a job from `needs`, or does + anything but check out the repository and run `require-results` on its + `needs` context; +- a `select-countries` job that doesn't publish the outputs its `plan` step + writes; +- `continue-on-error` on any job or step, a step `if` other than `always()`, + `success()` or `!cancelled()`, and a job-level `name`; +- a workflow `defaults` other than `run: shell: bash`, a job-level `defaults` + and a step `shell`. GitHub runs an explicit `shell: bash` as + `bash -eo pipefail`, so a pipe cannot hide a failed command. + +The plan's tests in `test_ci_test_plan.py` run the same checks and test the +code `ci` runs. The ruleset reads `engine-free` without `ci`, so those tests +stay in that job, and `tools/ci_test_plan.py verify` refuses a plan that moves +them out. After editing the workflow, run +`uv run python tools/ci_test_plan.py verify-workflow`. A pull request's run +takes the workflow and the plan from the pull request's own merge with `main`, +so a pull request that rewrites them can pass itself, and review has to catch +that. + `tools/ci_test_plan.py` is the only authority for test-directory ownership, CI job assignment, country ownership, US timing-report categories, and changed-path country selection. Its `TEST_GROUPS` registry defines every valid @@ -84,9 +132,9 @@ test in a directory registered by `TEST_GROUPS` and must run through that category's existing workflow job. Supporting programs and data may live under `tools/`, but they do not receive separate workflow jobs. The test-plan verifier allows only registered test jobs and the explicitly declared orchestration, -lint, and wheel-building jobs. Add a test category to `TEST_GROUPS` before its -executor; do not create a separate selection system, test workflow, or -single-purpose behavioral test job. +lint, wheel-building, and aggregate-result jobs. Add a test category to +`TEST_GROUPS` before its executor; do not create a separate selection system, +test workflow, or single-purpose behavioral test job. The Orrery parser compatibility assertion lives in `packages/microcosm-graph/tests/engine_free/shared/test_graph_orrery.py`. The diff --git a/packages/microcosm-build/tests/engine_free/shared/test_ci_integration_job.py b/packages/microcosm-build/tests/engine_free/shared/test_ci_integration_job.py index f73baf4898..62e69875a1 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_ci_integration_job.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_ci_integration_job.py @@ -68,7 +68,7 @@ def test_integration_job_is_uk_selected_and_read_only() -> None: assert "needs: select-countries" in integration assert "if: needs['select-countries'].outputs.run_uk == 'true'" in integration - assert "timeout-minutes: 15" in integration + assert "timeout-minutes: 20" in integration assert "permissions:\n contents: read" in integration assert "persist-credentials: false" in integration assert "environment:" not in integration diff --git a/packages/microcosm-build/tests/engine_free/shared/test_ci_test_plan.py b/packages/microcosm-build/tests/engine_free/shared/test_ci_test_plan.py index 7ef9c92bfd..7a0d8a6256 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_ci_test_plan.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_ci_test_plan.py @@ -2,9 +2,19 @@ from __future__ import annotations +import dataclasses +import itertools +import json +import os +import shlex +import subprocess +import sys from pathlib import Path import pytest +import yaml +from hypothesis import HealthCheck, event, given, settings +from hypothesis import strategies as st from tools import ci_test_plan @@ -178,6 +188,1590 @@ def test_workflow_rejects_an_underscore_prefixed_job() -> None: ) +def _other_jobs(source: str) -> tuple[str, ...]: + return tuple( + name + for name in ci_test_plan.workflow_job_names(source) + if name != ci_test_plan.AGGREGATE_JOB + ) + + +def _document_errors(source: str) -> tuple[str, ...]: + return ci_test_plan.workflow_document_errors(ci_test_plan.load_workflow(source)) + + +def test_aggregate_job_speaks_for_every_other_workflow_job() -> None: + source = ci_test_plan.WORKFLOW.read_text(encoding="utf-8") + jobs = yaml.safe_load(source)["jobs"] + + assert _document_errors(source) == () + assert sorted(jobs["ci"]["needs"]) == sorted(_other_jobs(source)) + assert sorted(jobs["ci"]["needs"]) == sorted(ci_test_plan.plan_jobs()) + + +def test_the_selection_decides_only_the_country_engine_jobs() -> None: + jobs = yaml.safe_load(ci_test_plan.WORKFLOW.read_text(encoding="utf-8"))["jobs"] + + outputs = {job: ci_test_plan.selection_output(job) for job in jobs if job != "ci"} + + assert outputs == { + "select-countries": None, + "lint": None, + "engine-free": None, + "engine-us": "run_us", + "engine-uk": "run_uk", + "integration-uk": "run_uk", + "wheels": None, + "installed-emitter": None, + } + assert ( + ci_test_plan.selection_condition("engine-us") + == "needs['select-countries'].outputs.run_us == 'true'" + ) + assert ci_test_plan.selection_condition("lint") is None + assert ci_test_plan.selection_output("no-such-job") is None + + +def test_the_selection_decides_a_job_only_when_every_group_needs_one_engine( + monkeypatch: pytest.MonkeyPatch, +) -> None: + group = ci_test_plan.TestGroup + monkeypatch.setattr( + ci_test_plan, + "TEST_GROUPS", + { + "one-engine": group(("a", "us"), "one", "us", engine_module="us_engine"), + "mixed-engine": group( + ("b", "us"), "mixed", "us", engine_module="us_engine" + ), + "mixed-free": group(("c", "us"), "mixed", "us"), + "two-us": group(("d", "us"), "two", "us", engine_module="us_engine"), + "two-uk": group(("e", "uk"), "two", "uk", engine_module="uk_engine"), + }, + ) + + assert ci_test_plan.selection_output("one") == "run_us" + assert ci_test_plan.selection_output("mixed") is None + assert ci_test_plan.selection_output("two") is None + + +def test_plan_refuses_a_job_that_waits_on_an_output_the_selection_never_writes( + monkeypatch: pytest.MonkeyPatch, +) -> None: + assert ci_test_plan.selection_errors() == () + + monkeypatch.setattr(ci_test_plan, "select_countries", lambda paths: {"run_us": 1}) + + assert ci_test_plan.selection_errors() == ( + "engine-uk: the selection writes no run_uk output", + "integration-uk: the selection writes no run_uk output", + ) + + +def test_verify_reports_the_plans_selection_errors( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(ci_test_plan, "selection_errors", lambda: ("no such output",)) + monkeypatch.setattr(ci_test_plan, "test_files", lambda: (Path(__file__),)) + + with pytest.raises(SystemExit) as refused: + ci_test_plan.verify() + + assert "\n no such output\n" in str(refused.value) + + +def test_verify_reports_the_plan_tests_errors( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(ci_test_plan, "plan_tests_errors", lambda: ("tests moved",)) + monkeypatch.setattr(ci_test_plan, "test_files", lambda: (Path(__file__),)) + + with pytest.raises(SystemExit) as refused: + ci_test_plan.verify() + + assert "\n tests moved\n" in str(refused.value) + + +_PLAN_TESTS_MOVED = ( + "packages/microcosm-build/tests/engine_free/shared/test_ci_test_plan.py: " + "engine-free must run this file, the plan's own tests, on every pull request, " + "because the main ruleset reads that job without ci. If the tests moved, " + "update PLAN_TESTS", +) + + +def test_engine_free_runs_these_tests() -> None: + """The ``main`` ruleset names both ``engine-free`` legs, so it reads that + job without ``ci``. These tests check the code ``ci`` runs, and they stay + in that job.""" + + assert ( + Path(__file__).resolve() + == (ci_test_plan.ROOT / ci_test_plan.PLAN_TESTS).resolve() + ) + assert ci_test_plan.selection_output(ci_test_plan.PLAN_TESTS_JOB) is None + assert ci_test_plan.plan_tests_errors() == () + + +@pytest.mark.parametrize( + "moved", + [ + {"job": "engine-us"}, + {"engine_module": "policyengine_us"}, + {"integration": True}, + ], +) +def test_plan_refuses_tests_that_engine_free_would_not_run( + monkeypatch: pytest.MonkeyPatch, moved: dict[str, object] +) -> None: + group = ci_test_plan.TEST_GROUPS["engine-free-shared"] + monkeypatch.setitem( + ci_test_plan.TEST_GROUPS, + "engine-free-shared", + dataclasses.replace(group, **moved), + ) + + assert ci_test_plan.plan_tests_errors() == _PLAN_TESTS_MOVED + + +def test_plan_refuses_tests_that_left_their_directory( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + here = ci_test_plan.PLAN_TESTS + monkeypatch.setattr(ci_test_plan, "ROOT", tmp_path) + monkeypatch.setattr(ci_test_plan, "PACKAGES", tmp_path / "packages") + for path, stays in ( + (here, True), + (here.replace("engine_free/shared", "engine_free/uk"), True), + (here.replace("engine_free/shared", "engine_contract/us"), False), + (here.replace("engine_free/shared", "engine_free"), False), + # No job runs a test outside ``packages/*/tests``. + (here.replace("packages/microcosm-build/tests", "tools/tests"), False), + ): + (tmp_path / path).parent.mkdir(parents=True, exist_ok=True) + (tmp_path / path).write_text("", encoding="utf-8") + monkeypatch.setattr(ci_test_plan, "PLAN_TESTS", path) + assert ci_test_plan.plan_tests_errors() == ( + () if stays else (_PLAN_TESTS_MOVED[0].replace(here, path),) + ), path + + missing = here.replace("test_ci_test_plan.py", "test_no_such_file.py") + monkeypatch.setattr(ci_test_plan, "PLAN_TESTS", missing) + assert ci_test_plan.plan_tests_errors() == ( + _PLAN_TESTS_MOVED[0].replace(here, missing), + ) + + +def test_plan_refuses_tests_in_a_job_the_selection_may_leave_out( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(ci_test_plan, "selection_output", lambda job: "run_us") + + assert ci_test_plan.plan_tests_errors() == _PLAN_TESTS_MOVED + + +def test_this_file_holds_the_tests_of_what_ci_runs() -> None: + """``verify`` keeps this file in ``engine-free``. This keeps the tests of + ``require-results`` in this file, where that job runs them.""" + + for name in ( + "test_require_results_fails_a_failed_run_when_ci_runs_it_as_a_script", + "test_require_results_reads_the_needs_context_for_the_real_workflow", + "test_require_results_hears_from_every_job_the_workflow_declares", + "test_aggregate_passes_exactly_when_every_job_passed_or_was_left_out", + "test_aggregate_rule_holds_for_every_combination_of_results", + ): + assert callable(globals().get(name)), name + + +def test_the_workflow_reports_the_checks_the_main_ruleset_names() -> None: + """The ``main`` ruleset names checks that run on every pull request, and a + matrix leg's name holds its value. When a job this test names changes, + change the ruleset too.""" + + jobs = yaml.safe_load(ci_test_plan.WORKFLOW.read_text(encoding="utf-8"))["jobs"] + checks: set[str] = set() + for name, job in jobs.items(): + if ci_test_plan.selection_output(name) is not None: + continue + matrix = job.get("strategy", {}).get("matrix") + if matrix is None: + checks.add(name) + else: + assert len(matrix) == 1, f"{name}: name its checks here and in the ruleset" + (legs,) = matrix.values() + checks.update(f"{name} ({leg})" for leg in legs) + + assert checks == { + "ci", + "engine-free (3.13)", + "engine-free (3.14)", + "installed-emitter", + "lint", + "select-countries", + "wheels", + }, "the main ruleset names these checks: change it with the job that reports them" + + +def test_lint_verifies_the_plan_and_the_repository_workflow() -> None: + """``verify`` alone checks that ``engine-free`` runs these tests.""" + + lint = yaml.safe_load(ci_test_plan.WORKFLOW.read_text(encoding="utf-8"))["jobs"][ + "lint" + ] + runs = [step.get("run") for step in lint["steps"]] + + assert "python3 tools/ci_test_plan.py verify" in runs + assert "uv run --no-sync python tools/ci_test_plan.py verify-workflow" in runs + + +_AGGREGATE_WORKFLOW = """\ +name: Tests +on: + pull_request: + branches: [main] +defaults: + run: + shell: bash +jobs: + select-countries: + outputs: + run_us: ${{ steps.plan.outputs.run_us }} + run_uk: ${{ steps.plan.outputs.run_uk }} + steps: + - uses: actions/checkout@v4 + - id: plan + run: python3 tools/ci_test_plan.py select-countries + lint: + steps: + - run: uv run --no-sync ruff check . + engine-free: + steps: + - run: pytest + engine-us: + needs: select-countries + if: needs['select-countries'].outputs.run_us == 'true' + steps: + - run: pytest + engine-uk: + needs: [select-countries] + if: needs['select-countries'].outputs.run_uk == 'true' + steps: + - run: pytest + integration-uk: + needs: + - select-countries + if: needs['select-countries'].outputs.run_uk == 'true' + steps: + - run: pytest + wheels: + steps: + - run: build + installed-emitter: + steps: + - run: pytest + ci: + if: always() + needs: + - select-countries + - lint + - engine-free + - engine-us + - engine-uk + - integration-uk + - wheels + - installed-emitter + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - env: + NEEDS: ${{ toJSON(needs) }} + run: python3 tools/ci_test_plan.py require-results +""" +_WHEELS = " wheels:\n steps:\n - run: build\n" +_CI_IF = " ci:\n if: always()\n" +_US_IF = " if: needs['select-countries'].outputs.run_us == 'true'\n" +_CI_CHECKOUT = ( + " runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v4\n" +) +_RUN_LINE = " run: python3 tools/ci_test_plan.py require-results\n" +_RUN_STEP = " - env:\n NEEDS: ${{ toJSON(needs) }}\n" + _RUN_LINE +_OUTPUTS_BLOCK = ( + " outputs:\n run_us: ${{ steps.plan.outputs.run_us }}\n" + " run_uk: ${{ steps.plan.outputs.run_uk }}\n" +) +_PLAN_STEP = ( + " - id: plan\n run: python3 tools/ci_test_plan.py select-countries\n" +) +_CI_MUST_ALWAYS_RUN = ( + "ci: `if` must read always(), because GitHub reports a skipped required " + "check as passing" +) +_US_MUST_READ = ( + "engine-us: `if` must read needs['select-countries'].outputs.run_us == 'true'" +) +_CI_STEPS = ( + "ci: steps must be a checkout that sets only `name`, `uses` and `with`, then " + "one `run` step" +) +_CI_RUN = ( + "ci: its `run` step must set `run` to `python3 tools/ci_test_plan.py " + "require-results` on one line, `env` to NEEDS: ${{ toJSON(needs) }} alone, and " + "no other key but `name`" +) +_CI_KEYS = ( + "ci: job may set only if, needs, permissions, runs-on, timeout-minutes, steps" +) +_PLAN_STEP_MUST_RUN = ( + "select-countries: one step with id `plan` must run `python3 " + "tools/ci_test_plan.py select-countries` on one line" +) +_STEP_IF = ( + "a step sets an `if` other than `always()`, `success()` or `!cancelled()`. " + "Write `if: always()`, `if: ${{ !cancelled() }}` or no `if`, so the step runs " + "whenever its job passes" +) +_SHELL = ( + "job sets `defaults` or a step sets `shell`, which can replace the workflow's " + "`bash -eo pipefail`" +) +_WORKFLOW_DEFAULTS = "defaults:\n run:\n shell: bash\n" +_WORKFLOW_SHELL = ( + "workflow: `defaults` must set `run: shell: bash` and nothing else, so every " + "`run` step runs under `bash -eo pipefail` and a pipe cannot hide a failed " + "command", +) +_JOB_CONTINUES = ( + "engine-free: job sets `continue-on-error`, which can let the run pass when " + "the job fails" +) +_OUTPUTS_MUST_MAP = ( + "select-countries: outputs must map each of run_us, run_uk to " + "`${{ steps.plan.outputs. }}`" +) +_NO_IF = ( + "lint: job must have no `if`, because the plan runs it on every run and ci " + "refuses its skip" +) +_NOT_DECLARED = "the test plan doesn't declare this workflow job" + + +def _edited(old: str, new: str) -> str: + assert _AGGREGATE_WORKFLOW.count(old) == 1, old + return _AGGREGATE_WORKFLOW.replace(old, new) + + +def test_a_complete_aggregate_job_has_no_workflow_errors() -> None: + assert _document_errors(_AGGREGATE_WORKFLOW) == () + assert ci_test_plan.workflow_job_errors(_AGGREGATE_WORKFLOW) == () + + +@pytest.mark.parametrize( + ("old", "new", "expected"), + [ + # ``ci`` must hear from every job. + (" - engine-us\n", "", ("ci: needs must list engine-us",)), + ( + " - wheels\n", + " - wheels\n - docs\n", + ("ci: needs lists docs, which the workflow does not declare",), + ), + ( + _WHEELS, + "", + ( + "wheels: the workflow lacks this job of the test plan", + "ci: needs lists wheels, which the workflow does not declare", + ), + ), + # A job outside the plan, however YAML spells it. + ( + _WHEELS, + _WHEELS + " deploy:\n steps:\n - run: ship\n", + (f"deploy: {_NOT_DECLARED}", "ci: needs must list deploy"), + ), + ( + _WHEELS, + _WHEELS + ' "deploy": {steps: [{run: ship}]}\n', + (f"deploy: {_NOT_DECLARED}", "ci: needs must list deploy"), + ), + ( + _WHEELS, + _WHEELS + " yes:\n steps:\n - run: ship\n", + (f"True: {_NOT_DECLARED}", "ci: needs must list True"), + ), + # ``ci`` must run whatever the other jobs report. + (_CI_IF, " ci:\n", (_CI_MUST_ALWAYS_RUN,)), + (_CI_IF, " ci:\n if: success()\n", (_CI_MUST_ALWAYS_RUN,)), + (_CI_IF, " ci:\n if: ${{ !cancelled() }}\n", (_CI_MUST_ALWAYS_RUN,)), + (_CI_IF, " ci:\n if: ${{ always() }}\n", (_CI_MUST_ALWAYS_RUN,)), + (_CI_IF, " ci:\n if:\n", (_CI_MUST_ALWAYS_RUN,)), + (_CI_IF, " ci:\n if: true\n", (_CI_MUST_ALWAYS_RUN,)), + # An ``if`` that continues on the next line, as a label bypass might. + (_CI_IF, " ci:\n if: always()\n && false\n", (_CI_MUST_ALWAYS_RUN,)), + ( + _CI_IF, + " ci:\n if: always()\n" + " && !contains(github.event.pull_request.labels.*.name, 'skip-ci')\n", + (_CI_MUST_ALWAYS_RUN,), + ), + (_CI_IF, f" ci:\n if: always(){chr(0xA0)}\n", (_CI_MUST_ALWAYS_RUN,)), + # ``ci`` does nothing but hand the results to ``require-results``. + (_RUN_LINE, _RUN_LINE.replace("\n", " || true\n"), (_CI_RUN,)), + # A block scalar ends in a newline, so the command no longer matches. + ( + _RUN_LINE, + " run: |\n python3 tools/ci_test_plan.py require-results\n", + (_CI_RUN,), + ), + ( + " NEEDS: ${{ toJSON(needs) }}\n", + " RESULTS: ${{ toJSON(needs) }}\n", + (_CI_RUN,), + ), + ( + " NEEDS: ${{ toJSON(needs) }}\n", + " NEEDS: ${{ toJSON(needs) }}\n PYTHONPATH: elsewhere\n", + (_CI_RUN,), + ), + ( + " - env:\n NEEDS: ${{ toJSON(needs) }}\n", + " -\n", + (_CI_RUN,), + ), + ( + " - env:\n NEEDS: ${{ toJSON(needs) }}\n", + " - env: NEEDS\n", + (_CI_RUN,), + ), + (_RUN_STEP, _RUN_STEP + " shell: true {0}\n", (_CI_RUN,)), + (_RUN_STEP, _RUN_STEP + " if: false\n", (_CI_RUN,)), + (_RUN_STEP, _RUN_STEP + " continue-on-error: true\n", (_CI_RUN,)), + (_RUN_STEP, "", (_CI_STEPS,)), + (_RUN_STEP, _RUN_STEP + " - run: echo done\n", (_CI_STEPS,)), + (_RUN_STEP, " - run: echo first\n" + _RUN_STEP, (_CI_STEPS,)), + ( + _CI_CHECKOUT, + _CI_CHECKOUT.replace("actions/checkout@v4", "someone/rewrite@v1"), + (_CI_STEPS,), + ), + (_CI_CHECKOUT, _CI_CHECKOUT + " if: false\n", (_CI_STEPS,)), + ( + _CI_CHECKOUT + _RUN_STEP, + " runs-on: ubuntu-latest\n steps: checkout\n", + ("ci: steps must be a list of mappings", _CI_STEPS), + ), + # ``ci`` sets no key that would rename its check or change its steps. + ( + _CI_IF, + _CI_IF + " strategy:\n matrix:\n leg: [a, b]\n", + (_CI_KEYS,), + ), + ( + _CI_IF, + _CI_IF + " defaults:\n run:\n shell: true {0}\n", + (_CI_KEYS,), + ), + (_CI_IF, _CI_IF + " env:\n PYTHONPATH: elsewhere\n", (_CI_KEYS,)), + (_CI_IF, _CI_IF + " continue-on-error: true\n", (_CI_KEYS,)), + (_CI_IF, _CI_IF + " name: Required\n", (_CI_KEYS,)), + # The selection job publishes the outputs its step writes. + ( + " run_us: ${{ steps.plan.outputs.run_us }}\n", + " run_us: ${{ steps.plan.outputs.run_uk }}\n", + (_OUTPUTS_MUST_MAP,), + ), + (" run_uk: ${{ steps.plan.outputs.run_uk }}\n", "", (_OUTPUTS_MUST_MAP,)), + (_OUTPUTS_BLOCK, " outputs: none\n", (_OUTPUTS_MUST_MAP,)), + (_OUTPUTS_BLOCK, "", (_OUTPUTS_MUST_MAP,)), + (_PLAN_STEP, "", (_PLAN_STEP_MUST_RUN,)), + ( + _PLAN_STEP, + _PLAN_STEP + " if: false\n", + (f"select-countries: {_STEP_IF}",), + ), + ( + _PLAN_STEP, + " - id: plan\n run: echo run_us=false >> $GITHUB_OUTPUT\n", + (_PLAN_STEP_MUST_RUN,), + ), + ( + _PLAN_STEP, + " - id: plan\n run: |\n" + " python3 tools/ci_test_plan.py select-countries\n", + (_PLAN_STEP_MUST_RUN,), + ), + (_PLAN_STEP, _PLAN_STEP + _PLAN_STEP, (_PLAN_STEP_MUST_RUN,)), + # Each other job carries the ``if`` the plan gives it and no other. + (_US_IF, "", (_US_MUST_READ,)), + (_US_IF, " if: always()\n", (_US_MUST_READ,)), + (_US_IF, " if: github.event_name == 'push'\n", (_US_MUST_READ,)), + ( + _US_IF, + " if: ${{ needs['select-countries'].outputs.run_us == 'true' }}\n", + (_US_MUST_READ,), + ), + ( + _US_IF, + " if: needs['select-countries'].outputs.run_us != 'false'\n", + (_US_MUST_READ,), + ), + ( + _US_IF, + " if: needs['select-countries'].outputs.run_uk == 'true'\n", + (_US_MUST_READ,), + ), + (_US_IF, _US_IF.rstrip("\n") + " || always()\n", (_US_MUST_READ,)), + (_US_IF, _US_IF + " && false\n", (_US_MUST_READ,)), + (" lint:\n", " lint:\n if: github.event_name == 'push'\n", (_NO_IF,)), + (" lint:\n", " lint:\n if:\n", (_NO_IF,)), + ( + " needs: select-countries\n", + " needs: lint\n", + ("engine-us: needs must list select-countries",), + ), + ( + " needs: select-countries\n", + " needs: {job: select-countries}\n", + ( + "engine-us: needs must be a job name or a list of job names", + "engine-us: needs must list select-countries", + ), + ), + # A job that needs a job the selection may leave out skips with it. + ( + " wheels:\n", + " wheels:\n needs: engine-uk\n", + ( + "wheels: needs engine-uk, which the selection may leave out when " + "it runs wheels, and GitHub would then skip wheels too", + ), + ), + ( + " needs: select-countries\n", + " needs: [select-countries, integration-uk]\n", + ( + "engine-us: needs integration-uk, which the selection may leave " + "out when it runs engine-us, and GitHub would then skip " + "engine-us too", + ), + ), + # Keys and steps that would hide a failure or rename a check. + ( + " engine-free:\n", + " engine-free:\n continue-on-error: true\n", + (_JOB_CONTINUES,), + ), + ( + " engine-free:\n", + " engine-free:\n continue-on-error: ${{ matrix.experimental }}\n", + (_JOB_CONTINUES,), + ), + # PyYAML reads ``no`` as false, and YAML 1.2 reads a string. + ( + " engine-free:\n", + " engine-free:\n continue-on-error: no\n", + (_JOB_CONTINUES,), + ), + ( + " engine-free:\n", + ' engine-free:\n "name": ci\n', + ( + "engine-free: job sets `name`, which would report the job under " + "another check name", + ), + ), + ( + " - run: build\n", + " - run: build\n continue-on-error: true\n", + ( + "wheels: a step sets `continue-on-error`, which can let the job " + "pass when the step fails", + ), + ), + # A step that can skip leaves its job passing with its tests unrun. + ( + " engine-free:\n steps:\n - run: pytest\n", + " engine-free:\n steps:\n - run: pytest\n" + " if: github.event_name == 'push'\n", + (f"engine-free: {_STEP_IF}",), + ), + ( + " - run: build\n", + " - run: build\n if: failure()\n", + (f"wheels: {_STEP_IF}",), + ), + ( + _US_IF + " steps:\n - run: pytest\n", + _US_IF + " steps:\n - run: pytest\n" + " if: github.event_name == 'push'\n", + (f"engine-us: {_STEP_IF}",), + ), + # A step inside a group of steps counts too. + ( + " - run: build\n", + " - parallel:\n - run: build\n" + " if: github.event_name == 'push'\n", + (f"wheels: {_STEP_IF}",), + ), + ( + " - run: build\n", + " - parallel:\n - run: build\n" + " continue-on-error: true\n", + ( + "wheels: a step sets `continue-on-error`, which can let the job " + "pass when the step fails", + ), + ), + # PyYAML reads ``no`` as false, and YAML 1.2 reads a string. + ( + " - run: build\n", + " - run: build\n continue-on-error: no\n", + ( + "wheels: a step sets `continue-on-error`, which can let the job " + "pass when the step fails", + ), + ), + ( + " - run: build\n", + " - run: build\n if: ${{ failure() }}\n", + (f"wheels: {_STEP_IF}",), + ), + ( + " - run: build\n", + " - run: build\n if: false\n", + (f"wheels: {_STEP_IF}",), + ), + ( + " - run: build\n", + " - run: build\n if: cancelled()\n", + (f"wheels: {_STEP_IF}",), + ), + ( + " - run: build\n", + " - run: build\n if: always() && github.event_name == 'push'\n", + (f"wheels: {_STEP_IF}",), + ), + # Every ``run`` step runs under ``bash -eo pipefail``, where a pipe + # cannot hide the exit status of the command that feeds it. + (_WORKFLOW_DEFAULTS, "", _WORKFLOW_SHELL), + (_WORKFLOW_DEFAULTS, "defaults:\n run:\n shell: sh\n", _WORKFLOW_SHELL), + ( + _WORKFLOW_DEFAULTS, + _WORKFLOW_DEFAULTS + " working-directory: tools\n", + _WORKFLOW_SHELL, + ), + ( + " wheels:\n", + " wheels:\n defaults:\n run:\n shell: sh\n", + (f"wheels: {_SHELL}",), + ), + ( + " - run: build\n", + " - run: build\n shell: true {0}\n", + (f"wheels: {_SHELL}",), + ), + ( + " - run: build\n", + " - parallel:\n - run: build\n shell: sh\n", + (f"wheels: {_SHELL}",), + ), + (_WHEELS, " wheels: build\n", ("wheels: job must be a mapping",)), + ( + _WHEELS, + " wheels:\n steps:\n - build\n", + ("wheels: steps must be a list of mappings",), + ), + ], +) +def test_workflow_rejects_a_job_whose_failure_could_pass_ci( + old: str, new: str, expected: tuple[str, ...] +) -> None: + assert _document_errors(_edited(old, new)) == expected + + +@pytest.mark.parametrize( + ("old", "new"), + [ + # Other spellings of the same YAML. + (" wheels:\n", ' "wheels":\n'), + (" wheels:\n", " wheels: # builds every wheel\n"), + (_CI_IF, ' ci:\n if: "always()"\n'), + (_CI_IF, " ci:\n if: always() # even when a job fails\n"), + (_CI_IF, " ci:\n if: >-\n always()\n"), + (_CI_IF, " ci:\n timeout-minutes: 5\n if: always()\n"), + ( + " needs: select-countries\n" + _US_IF, + _US_IF + " needs: [select-countries, lint] # selection\n", + ), + (" - wheels\n", " - 'wheels' # archives\n"), + (" lint:\n", " lint: &lint\n"), + # A job may need jobs that run whenever it runs. + (" wheels:\n", " wheels:\n needs: [lint, engine-free]\n"), + (" lint:\n", " lint:\n needs: wheels\n"), + ( + " needs: [select-countries]\n", + " needs: [select-countries, integration-uk]\n", + ), + # Keys and steps a job may carry. + ( + " engine-free:\n steps:\n", + " engine-free:\n runs-on: ubuntu-latest\n timeout-minutes: 15\n" + " permissions:\n contents: read\n" + " env:\n NO: a\n YES: b\n strategy:\n" + " fail-fast: false\n matrix:\n" + ' python-version: ["3.13", "3.14"]\n steps:\n', + ), + ( + " - run: build\n", + " - name: Build\n if: ${{ !cancelled() }}\n" + " run: |\n echo \"it's [not closed\n jobs:\n", + ), + ( + _CI_CHECKOUT, + " runs-on: ubuntu-latest\n permissions:\n contents: read\n" + " steps:\n - name: Check out\n uses: actions/checkout@v5\n" + " with:\n persist-credentials: false\n", + ), + ( + _RUN_STEP, + " - name: Require every job to pass\n " + _RUN_STEP[8:], + ), + # A step may carry a condition that still runs it in a passing job. + (" - run: build\n", " - run: build\n if: always()\n"), + (" - run: build\n", " - run: build\n if: ${{ always() }}\n"), + (" - run: build\n", " - run: build\n if: success()\n"), + (" - run: build\n", ' - run: build\n if: "!cancelled()"\n'), + # A pipe can't hide a failure under the workflow's ``bash -eo pipefail``. + (" - run: build\n", " - run: build | tee build.log\n"), + # An action may take an input that shares a step key's name. + ( + " - run: build\n", + " - uses: actions/setup-node@v7\n with:\n" + " shell: sh\n if: never\n" + " continue-on-error: true\n - run: build\n", + ), + # The selection job may publish more, and its step may carry more. + ( + _PLAN_STEP, + " - name: Select country tests\n id: plan\n env:\n" + " GITHUB_TOKEN: ${{ github.token }}\n" + " run: python3 tools/ci_test_plan.py select-countries\n" + " - run: echo done\n", + ), + ( + " run_uk: ${{ steps.plan.outputs.run_uk }}\n", + " run_uk: ${{ steps.plan.outputs.run_uk }}\n" + " docs_only: ${{ steps.plan.outputs.docs_only }}\n", + ), + ], +) +def test_workflow_accepts_other_spellings_and_ordinary_edits( + old: str, new: str +) -> None: + assert _document_errors(_edited(old, new)) == () + + +@pytest.mark.parametrize( + "document", + [None, "jobs", [], {}, {"jobs": None}, {"jobs": ["lint"]}, {"on": "push"}], +) +def test_workflow_rejects_a_document_with_no_jobs_mapping(document: object) -> None: + assert ci_test_plan.workflow_document_errors(document) == ( + "workflow has no jobs mapping", + ) + + +@pytest.mark.parametrize( + ("old", "new", "refusal"), + [ + # PyYAML alone would keep the second ``if`` and say nothing. + ( + _CI_IF, + " ci:\n if: false\n if: always()\n", + "a mapping sets `if` twice", + ), + ( + _CI_IF, + ' ci:\n "if": false\n if: always()\n', + "a mapping sets `if` twice", + ), + ( + _CI_IF, + ' ci:\n if: false\n "if": always()\n', + "a mapping sets `if` twice", + ), + (_WHEELS, _WHEELS + _WHEELS, "a mapping sets `wheels` twice"), + ("jobs:\n", "jobs: {}\njobs:\n", "a mapping sets `jobs` twice"), + # PyYAML alone would build ``False`` from both keys and keep the last. + ( + " wheels:\n", + " wheels:\n env:\n NO: a\n OFF: b\n", + "PyYAML reads `NO` and `OFF` as one key", + ), + ( + _WHEELS, + _WHEELS + " yes:\n steps: []\n on:\n steps: []\n", + "PyYAML reads `yes` and `on` as one key", + ), + # PyYAML alone would merge the anchored mapping into ``ci``. + ( + _CI_IF, + " ci:\n <<: *lint\n if: always()\n", + "a mapping uses a merge key (`<<`)", + ), + ], +) +def test_load_workflow_refuses_what_github_may_read_differently( + old: str, new: str, refusal: str +) -> None: + source = _edited(old, new).replace(" lint:\n", " lint: &lint\n") + + assert isinstance(yaml.safe_load(source), dict) + with pytest.raises(ValueError) as refused: + ci_test_plan.load_workflow(source) + assert str(refused.value).startswith(f"workflow fails to load: {refusal}\n") + + +@pytest.mark.parametrize("code", [0x85, 0x2028, 0x2029]) +def test_load_workflow_names_a_line_break_that_only_yaml_1_1_reads(code: int) -> None: + """PyYAML alone would end the line here and read the rest as a comment.""" + + source = _edited(_CI_IF, f" ci:\n if: always(){chr(code)} # && false\n") + line = source[: source.index(chr(code))].count("\n") + 1 + + assert yaml.safe_load(source)["jobs"]["ci"]["if"] == "always()" + with pytest.raises(ValueError) as refused: + ci_test_plan.load_workflow(source) + assert str(refused.value) == ( + f"workflow line {line} holds U+{code:04X}, which PyYAML reads as a line " + "break and YAML 1.2 does not" + ) + + +@pytest.mark.parametrize( + ("source", "refusal"), + [ + ("jobs:\n lint: [\n", "while parsing a flow"), + ( + "jobs:\n ? [lint, wheels]\n : {}\n", + "a mapping uses a list or a mapping as a key", + ), + ], +) +def test_load_workflow_refuses_text_a_yaml_parser_refuses( + source: str, refusal: str +) -> None: + with pytest.raises(yaml.YAMLError): + yaml.safe_load(source) + with pytest.raises(ValueError) as refused: + ci_test_plan.load_workflow(source) + assert str(refused.value).startswith(f"workflow fails to load: {refusal}") + + +_ANCHORED = """\ +defaults: &defaults + runs-on: ubuntu-latest + env: + NO: a + YES: b +jobs: + lint: *defaults + wheels: *defaults +""" + + +def test_load_workflow_reads_what_a_yaml_parser_reads() -> None: + """Differential: the stricter loader against PyYAML's own.""" + + for source in ( + _ANCHORED, + ci_test_plan.WORKFLOW.read_text(encoding="utf-8"), + _AGGREGATE_WORKFLOW, + ): + assert ci_test_plan.load_workflow(source) == yaml.safe_load(source) + + +_VALID = yaml.safe_load(_AGGREGATE_WORKFLOW)["jobs"] +_JOB_NAMES = (*_VALID, "deploy") +_COUNTRY = {"engine-us": "us", "engine-uk": "uk", "integration-uk": "uk"} +_IFS = st.sampled_from( + [ + None, + "always()", + "success()", + "needs['select-countries'].outputs.run_us == 'true'", + "needs['select-countries'].outputs.run_uk == 'true'", + ] +) +_NEEDS = st.one_of( + st.none(), + st.sampled_from(["select-countries", "lint"]), + st.lists(st.sampled_from(_JOB_NAMES), unique=True, max_size=9), +) +_STEPS = st.sampled_from( + [ + *(job["steps"] for job in _VALID.values()), + [{"run": "pytest", "continue-on-error": False}], + [{"run": "pytest", "if": "always()"}], + [{"run": "pytest", "if": "failure()"}], + [{"run": "pytest", "if": "${{ !cancelled() }}"}], + [{"run": "pytest", "shell": "sh"}], + [{"parallel": [{"run": "pytest"}, {"run": "lint", "if": "always()"}]}], + [{"parallel": [{"run": "pytest", "continue-on-error": True}]}], + [_VALID["ci"]["steps"][1]], + [*_VALID["ci"]["steps"], {"run": "echo done"}], + [*_VALID["select-countries"]["steps"], {"id": "plan", "run": "echo"}], + [{**_VALID["select-countries"]["steps"][1], "if": "always()"}], + [{"id": "plan", "run": "echo run_us=false"}], + [], + ] +) +_SELECTION_OUTPUTS = st.sampled_from( + [ + None, + _VALID["select-countries"]["outputs"], + {**_VALID["select-countries"]["outputs"], "docs_only": "true"}, + {"run_us": "${{ steps.plan.outputs.run_uk }}", "run_uk": "true"}, + ] +) + + +@st.composite +def _job(draw: st.DrawFn) -> dict[str, object]: + """Draw a job from parts, most of which break a rule.""" + + job: dict[str, object] = {"steps": draw(_STEPS)} + for key, value in (("if", draw(_IFS)), ("needs", draw(_NEEDS))): + if value is not None: + job[key] = value + if draw(st.integers(min_value=0, max_value=2)) == 0: + job["outputs"] = draw(_SELECTION_OUTPUTS) + if draw(st.integers(min_value=0, max_value=9)) == 0: + key = draw( + st.sampled_from(["continue-on-error", "name", "strategy", "defaults"]) + ) + job[key] = "x" + return job + + +@st.composite +def _neutral(draw: st.DrawFn, name: str) -> dict[str, object]: + """Change the test workflow's job in ways no rule reads.""" + + job = dict(_VALID[name]) + if draw(st.booleans()): + job["timeout-minutes"] = 5 + if draw(st.booleans()): + job["permissions"] = {"contents": "read"} + steps = [dict(step) for step in job["steps"]] + if draw(st.booleans()): + steps = [{"name": "A step", **step} for step in steps] + if draw(st.booleans()): + steps = [ + {**step, "uses": "actions/checkout@v5", "with": {"fetch-depth": 1}} + if "uses" in step + else step + for step in steps + ] + if name != "ci" and draw(st.booleans()): + steps.append({"run": "echo done", "if": "always()"}) + job["steps"] = steps + if "needs" in job and draw(st.booleans()): + job["needs"] = list(reversed(_needed(job))) + if name == "select-countries" and draw(st.booleans()): + job["outputs"] = {**job["outputs"], "docs_only": "true"} + return job + + +def _needed(job: dict[str, object]) -> list[str]: + needs = job.get("needs", []) + return [needs] if isinstance(needs, str) else list(needs) + + +def _may_hide_a_failure(step: dict[str, object]) -> bool: + runs_while_passing = ( + "always()", + "success()", + "!cancelled()", + "${{ !cancelled() }}", + ) + return ( + "continue-on-error" in step + or "shell" in step + or step.get("if", "success()") not in runs_while_passing + or any(map(_may_hide_a_failure, step.get("parallel", []))) + ) + + +def _ci_passes(job: dict[str, object], others: set[str]) -> bool: + steps = job["steps"] + if job.get("if") != "always()" or set(_needed(job)) != others: + return False + allowed = {"if", "needs", "permissions", "runs-on", "timeout-minutes", "steps"} + if not set(job) <= allowed or len(steps) != 2: + return False + checkout, run = steps + return ( + str(checkout.get("uses")).startswith("actions/checkout@") + and set(checkout) <= {"name", "uses", "with"} + and run.get("run") == "python3 tools/ci_test_plan.py require-results" + and run.get("env") == {"NEEDS": "${{ toJSON(needs) }}"} + and set(run) <= {"name", "env", "run"} + ) + + +def _lets_ci_speak(jobs: dict[str, dict[str, object]]) -> bool: + """The rule for a workflow, stated independently of the implementation.""" + + if set(jobs) != set(_VALID): + return False + for name, job in jobs.items(): + steps = job["steps"] + if name == "ci": + if not _ci_passes(job, set(jobs) - {"ci"}): + return False + continue + if "continue-on-error" in job or "name" in job or "defaults" in job: + return False + if any(map(_may_hide_a_failure, steps)): + return False + wanted = ( + f"needs['select-countries'].outputs.run_{_COUNTRY[name]} == 'true'" + if name in _COUNTRY + else None + ) + if job.get("if") != wanted or ("if" in job) != (name in _COUNTRY): + return False + if name in _COUNTRY and "select-countries" not in _needed(job): + return False + if any( + _COUNTRY.get(needed, _COUNTRY.get(name)) != _COUNTRY.get(name) + for needed in _needed(job) + ): + return False + if name == "select-countries": + outputs = job.get("outputs") + plans = [step for step in steps if step.get("id") == "plan"] + if ( + not isinstance(outputs, dict) + or outputs.get("run_us") != "${{ steps.plan.outputs.run_us }}" + or outputs.get("run_uk") != "${{ steps.plan.outputs.run_uk }}" + or len(plans) != 1 + or plans[0].get("run") + != "python3 tools/ci_test_plan.py select-countries" + ): + return False + return True + + +@settings( + max_examples=800, + deadline=None, + derandomize=True, + suppress_health_check=[HealthCheck.too_slow], +) +@given(data=st.data()) +def test_workflow_passes_exactly_when_it_lets_ci_speak_for_every_job( + data: st.DataObject, +) -> None: + """Invariant: no errors iff the workflow declares the plan's jobs, none + hides a failure, ``ci`` always runs, needs every other job and only + requires their results, the selection job publishes what its step writes, + and each other job skips only as the plan says.""" + + jobs = { + name: data.draw(_job()) + for name in _JOB_NAMES + if data.draw(st.integers(min_value=0, max_value=19)) != 0 + } + # Most draws start from a workflow that passes, so both outcomes occur. + if data.draw(st.integers(min_value=0, max_value=3)) != 0: + jobs = {name: dict(job) for name, job in _VALID.items()} + for name in data.draw(st.lists(st.sampled_from(_JOB_NAMES), max_size=3)): + edit = data.draw( + st.sampled_from(["vary", "vary", "replace", "remove", "if", "needs"]) + ) + if edit == "remove": + jobs.pop(name, None) + elif edit == "vary" and name in _VALID: + jobs[name] = data.draw(_neutral(name)) + elif edit in ("vary", "replace") or name not in jobs: + jobs[name] = data.draw(_job()) + else: + value = data.draw(_IFS if edit == "if" else _NEEDS) + jobs[name] = {k: v for k, v in jobs[name].items() if k != edit} + if value is not None: + jobs[name][edit] = value + + errors = ci_test_plan.workflow_document_errors( + {"defaults": {"run": {"shell": "bash"}}, "jobs": jobs} + ) + + passes = _lets_ci_speak(jobs) + event( + "refused" + if not passes + else "passes, as the test workflow" + if jobs == _VALID + else "passes, changed" + ) + assert (errors == ()) is passes, errors + + +@settings( + max_examples=300, + deadline=None, + derandomize=True, + suppress_health_check=[HealthCheck.too_slow], +) +@given(data=st.data()) +def test_workflow_accepts_changes_no_rule_reads(data: st.DataObject) -> None: + """Invariant: a job keeps passing when it gains a timeout, permissions, + step names, another checkout version, a reordered ``needs``, an extra + step that always runs or an extra selection output.""" + + name = data.draw(st.sampled_from(sorted(_VALID))) + jobs = {**_VALID, name: data.draw(_neutral(name))} + + assert _lets_ci_speak(jobs) + assert ( + ci_test_plan.workflow_document_errors( + {"defaults": {"run": {"shell": "bash"}}, "jobs": jobs} + ) + == () + ) + + +def test_verify_workflow_passes_the_repository_workflow( + capsys: pytest.CaptureFixture[str], +) -> None: + assert ci_test_plan.main(["verify-workflow"]) == 0 + assert capsys.readouterr().out == "workflow=ok\n" + + +@pytest.mark.parametrize( + ("old", "new", "message"), + [ + ( + _CI_IF, + " ci:\n if: always()\n && false\n", + "workflow fails the test plan:\n " + _CI_MUST_ALWAYS_RUN, + ), + ( + _CI_IF, + " ci:\n if: always()\n if: always()\n", + "workflow fails to load: a mapping sets `if` twice", + ), + ], +) +def test_verify_workflow_refuses_a_workflow_the_plan_refuses( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, old: str, new: str, message: str +) -> None: + workflow = tmp_path / "test.yml" + workflow.write_text(_edited(old, new), encoding="utf-8") + monkeypatch.setattr(ci_test_plan, "WORKFLOW", workflow) + + with pytest.raises(SystemExit) as refused: + ci_test_plan.main(["verify-workflow"]) + + assert str(refused.value).startswith(message) + + +def _run_script( + *arguments: str, bare: bool = True, **environment: str +) -> subprocess.CompletedProcess[str]: + """Run the plan as the workflow does: as a script, from the repository root. + + ``bare`` adds ``-S``, which leaves site-packages out, as the runner's + ``python3`` lacks the workspace's packages. + """ + + return subprocess.run( + [sys.executable, *(["-S"] if bare else []), ci_test_plan.__file__, *arguments], + cwd=ci_test_plan.ROOT, + env=os.environ | environment, + capture_output=True, + text=True, + stdin=subprocess.DEVNULL, + check=False, + ) + + +def test_the_workflow_runs_this_module_as_a_script() -> None: + script = ["python3", "/".join(Path(ci_test_plan.__file__).parts[-2:])] + + assert script == ["python3", "tools/ci_test_plan.py"] + assert shlex.split(ci_test_plan.REQUIRE_RESULTS_COMMAND) == [ + *script, + "require-results", + ] + assert shlex.split(ci_test_plan.SELECTION_COMMAND) == [*script, "select-countries"] + + +def test_require_results_fails_a_failed_run_when_ci_runs_it_as_a_script() -> None: + """``ci`` passes or fails on this script's exit code, which no call of + ``main`` inside pytest can see.""" + + needs = _passing_needs() + passed = _run_script("require-results", NEEDS=json.dumps(needs)) + + assert (passed.returncode, passed.stderr) == (0, "") + assert passed.stdout.splitlines()[-1] == "results=ok" + + needs["engine-free"]["result"] = "failure" + failed = _run_script("require-results", NEEDS=json.dumps(needs)) + + assert failed.returncode == 1 + assert failed.stderr == "workflow did not pass:\n engine-free: failure\n" + + +def test_select_countries_writes_its_outputs_when_run_as_a_script( + tmp_path: Path, +) -> None: + event_path = tmp_path / "event.json" + event_path.write_text("{}", encoding="utf-8") + output = tmp_path / "output" + + run = _run_script( + "select-countries", + GITHUB_EVENT_NAME="push", + GITHUB_EVENT_PATH=str(event_path), + GITHUB_REPOSITORY="owner/repo", + GITHUB_TOKEN="", + GITHUB_OUTPUT=str(output), + ) + + assert (run.returncode, run.stderr) == (0, "") + assert output.read_text(encoding="utf-8") == "run_us=true\nrun_uk=true\n" + + +@pytest.mark.parametrize( + ("command", "bare", "last_line"), + [("verify", True, "verification=ok"), ("verify-workflow", False, "workflow=ok")], +) +def test_lint_commands_report_when_run_as_scripts( + command: str, bare: bool, last_line: str +) -> None: + run = _run_script(command, bare=bare) + + assert (run.returncode, run.stderr) == (0, "") + assert run.stdout.splitlines()[-1] == last_line + + +def _docs_listing(files: int) -> tuple[list[int], ci_test_plan.RequestJSON]: + """A pull request that changes ``files`` documentation files, 100 a page.""" + + pages: list[int] = [] + + def request_json(_url: str, _token: str) -> list[dict[str, str]]: + pages.append(len(pages) + 1) + listed = max(0, min(100, files - 100 * (len(pages) - 1))) + return [{"filename": "docs/guide.md"}] * listed + + return pages, request_json + + +def test_a_listing_that_reaches_githubs_limit_selects_both_countries() -> None: + """GitHub lists at most 3,000 files, so a full listing may leave files out.""" + + event = {"pull_request": {"number": 7}} + + pages, request_json = _docs_listing(2_950) + paths, selected = ci_test_plan.selection_for_event( + "pull_request", event, "owner/repo", "token", request_json=request_json + ) + assert len(paths) == 2_950 + assert selected == {"run_us": False, "run_uk": False} + + pages, request_json = _docs_listing(3_000) + paths, selected = ci_test_plan.selection_for_event( + "pull_request", event, "owner/repo", "token", request_json=request_json + ) + assert paths == [] + assert selected == {"run_us": True, "run_uk": True} + assert len(pages) == 30 + + +_JOBS = ("select-countries", "lint", "engine-free", "engine-us", "wheels") +_SELECTED_BY = {"engine-us": "run_us"} + + +def _needs(run_us: object = "true", **results: str) -> dict[str, dict[str, object]]: + base = dict.fromkeys(_JOBS, "success") | { + name.replace("_", "-"): result for name, result in results.items() + } + needs: dict[str, dict[str, object]] = { + job: {"result": result, "outputs": {}} for job, result in base.items() + } + if run_us is not None: + needs["select-countries"]["outputs"] = {"run_us": run_us} + return needs + + +def _errors(needs: dict[str, object]) -> tuple[str, ...]: + return ci_test_plan.aggregate_result_errors( + needs, jobs=_JOBS, selected_by=_SELECTED_BY + ) + + +def test_aggregate_passes_when_every_job_passed_or_was_left_out() -> None: + assert _errors(_needs()) == () + assert _errors(_needs(run_us="false", engine_us="skipped")) == () + # A job that ran and passed although the selection left it out still passed. + assert _errors(_needs(run_us="false")) == () + + +_NO_REPORT = ( + "engine-us: skipped, and select-countries did not report run_us as true or false", +) + + +@pytest.mark.parametrize( + ("needs", "expected"), + [ + (_needs(engine_free="failure"), ("engine-free: failure",)), + (_needs(engine_us="cancelled"), ("engine-us: cancelled",)), + (_needs(wheels="timed_out"), ("wheels: timed_out",)), + ( + _needs(lint="skipped"), + ("lint: skipped, but the plan runs it on every run",), + ), + ( + _needs(run_us="true", engine_us="skipped"), + ("engine-us: skipped, but select-countries selected it (run_us=true)",), + ), + (_needs(run_us=None, engine_us="skipped"), _NO_REPORT), + (_needs(run_us="", engine_us="skipped"), _NO_REPORT), + (_needs(run_us="False", engine_us="skipped"), _NO_REPORT), + (_needs(run_us="no", engine_us="skipped"), _NO_REPORT), + (_needs(run_us=False, engine_us="skipped"), _NO_REPORT), + ( + _needs(engine_us="skipped") + | {"select-countries": {"result": "success", "outputs": "run_us=false"}}, + _NO_REPORT, + ), + ( + _needs(run_us="false", select_countries="failure", engine_us="skipped"), + ( + "select-countries: failure", + "engine-us: skipped, and select-countries did not succeed", + ), + ), + ( + {job: entry for job, entry in _needs().items() if job != "wheels"}, + ("wheels: reported no result",), + ), + (_needs() | {"lint": "success"}, ("lint: reported no result",)), + ( + _needs() | {"docs": {"result": "success"}}, + ("docs: reported a result, but the plan declares no such job",), + ), + ({}, tuple(f"{job}: reported no result" for job in _JOBS)), + ], +) +def test_aggregate_names_each_job_that_did_not_pass( + needs: dict[str, object], expected: tuple[str, ...] +) -> None: + assert _errors(needs) == expected + + +# The same examples on every run, so a required check never fails by chance. +_PROPERTY = settings( + max_examples=300, + deadline=None, + derandomize=True, + suppress_health_check=[HealthCheck.too_slow], +) +_RESULT_VALUES = ("success", "failure", "cancelled", "skipped", "timed_out", None) +_RESULTS = st.sampled_from(_RESULT_VALUES) +_RUN_US = st.one_of( + st.none(), st.sampled_from(["true", "false"]), st.text(), st.booleans() +) + + +def _passes(job: str, results: dict[str, str | None], run_us: object) -> bool: + """The rule, stated independently of the implementation.""" + + if results[job] == "success": + return True + return ( + results[job] == "skipped" + and job in _SELECTED_BY + and results["select-countries"] == "success" + and isinstance(run_us, str) + and run_us == "false" + ) + + +def _context( + results: dict[str, str | None], run_us: object +) -> dict[str, dict[str, object]]: + needs: dict[str, dict[str, object]] = { + job: {"result": result, "outputs": {}} + for job, result in results.items() + if result is not None + } + if run_us is not None and "select-countries" in needs: + needs["select-countries"]["outputs"] = {"run_us": run_us} + return needs + + +@_PROPERTY +@given(results=st.fixed_dictionaries({job: _RESULTS for job in _JOBS}), run_us=_RUN_US) +def test_aggregate_passes_exactly_when_every_job_passed_or_was_left_out( + results: dict[str, str | None], run_us: object +) -> None: + """Invariant: one error for each job that neither passed nor skipped + because the selection succeeded and reported ``false`` for it.""" + + errors = _errors(_context(results, run_us)) + + assert [error.split(":")[0] for error in errors] == [ + job for job in _JOBS if not _passes(job, results, run_us) + ] + + +@_PROPERTY +@given( + results=st.fixed_dictionaries({job: _RESULTS for job in _JOBS}), + run_us=_RUN_US, + job=st.sampled_from(_JOBS), + bad=st.sampled_from(["failure", "cancelled", "timed_out"]), +) +def test_no_context_passes_with_a_failed_job( + results: dict[str, str | None], run_us: object, job: str, bad: str +) -> None: + """Invariant: a failed, cancelled or timed-out job always fails the check.""" + + results = results | {job: bad} + + assert f"{job}: {bad}" in _errors(_context(results, run_us)) + + +def test_aggregate_rule_holds_for_every_combination_of_results() -> None: + """Every result of every job, crossed with five values of the output.""" + + checked = 0 + for combination in itertools.product(_RESULT_VALUES, repeat=len(_JOBS)): + results = dict(zip(_JOBS, combination, strict=True)) + for run_us in ("true", "false", "", "False", None): + errors = _errors(_context(results, run_us)) + assert [error.split(":")[0] for error in errors] == [ + job for job in _JOBS if not _passes(job, results, run_us) + ], (results, run_us) + checked += 1 + + assert checked == 6**5 * 5 + + +_NAMES = st.text(alphabet="abcdefgh-", min_size=1, max_size=5) +_OUTPUT_NAMES = st.sampled_from(["run_us", "run_uk", "run_nz"]) + + +@_PROPERTY +@given(data=st.data()) +def test_aggregate_rule_holds_for_any_plan(data: st.DataObject) -> None: + """Invariant: for any set of jobs and any selection, one error for each + job that neither passed nor was left out, and one for each job the plan + does not declare.""" + + selection = ci_test_plan.SELECTION_JOB + jobs = (selection, *data.draw(st.lists(_NAMES, max_size=6, unique=True))) + selected_by = data.draw(st.dictionaries(st.sampled_from(jobs), _OUTPUT_NAMES)) + results = {job: data.draw(_RESULTS) for job in jobs} + outputs = data.draw(st.dictionaries(_OUTPUT_NAMES, _RUN_US)) + strangers = data.draw(st.lists(_NAMES.filter(lambda name: name not in jobs))) + needs: dict[str, object] = { + job: {"result": result, "outputs": outputs if job == selection else {}} + for job, result in results.items() + if result is not None + } + needs |= {name: {"result": "success", "outputs": {}} for name in strangers} + + def passes(job: str) -> bool: + if results[job] == "success": + return True + value = outputs.get(selected_by.get(job, "")) + return ( + results[job] == "skipped" + and job in selected_by + and results[selection] == "success" + and isinstance(value, str) + and value == "false" + ) + + errors = ci_test_plan.aggregate_result_errors( + needs, jobs=jobs, selected_by=selected_by + ) + + assert [error.split(":")[0] for error in errors] == [ + *(job for job in jobs if not passes(job)), + *dict.fromkeys(strangers), + ] + + +def _passing_needs() -> dict[str, dict[str, object]]: + needs: dict[str, dict[str, object]] = { + job: {"result": "success", "outputs": {}} for job in ci_test_plan.plan_jobs() + } + needs["select-countries"]["outputs"] = {"run_us": "true", "run_uk": "false"} + needs["engine-uk"]["result"] = "skipped" + needs["integration-uk"]["result"] = "skipped" + return needs + + +def test_require_results_reads_the_needs_context_for_the_real_workflow( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + needs = _passing_needs() + monkeypatch.setenv("NEEDS", json.dumps(needs)) + + assert ci_test_plan.main(["require-results"]) == 0 + assert "engine-uk=skipped" in capsys.readouterr().out + + needs["lint"]["result"] = "skipped" + needs["engine-us"]["result"] = "skipped" + monkeypatch.setenv("NEEDS", json.dumps(needs)) + with pytest.raises(SystemExit) as refused: + ci_test_plan.main(["require-results"]) + assert str(refused.value) == ( + "workflow did not pass:\n" + " lint: skipped, but the plan runs it on every run\n" + " engine-us: skipped, but select-countries selected it (run_us=true)" + ) + + +def test_require_results_hears_from_every_job_the_workflow_declares( + monkeypatch: pytest.MonkeyPatch, +) -> None: + source = ci_test_plan.WORKFLOW.read_text(encoding="utf-8") + + assert sorted(ci_test_plan.plan_jobs()) == sorted(_other_jobs(source)) + for job in ci_test_plan.plan_jobs(): + needs = _passing_needs() + del needs[job] + monkeypatch.setenv("NEEDS", json.dumps(needs)) + with pytest.raises(SystemExit) as refused: + ci_test_plan.main(["require-results"]) + assert f" {job}: reported no result" in str(refused.value) + + +@pytest.mark.parametrize("value", [None, "", "not json", "[]", '"success"']) +def test_require_results_refuses_a_missing_or_malformed_needs_context( + monkeypatch: pytest.MonkeyPatch, value: str | None +) -> None: + if value is None: + monkeypatch.delenv("NEEDS", raising=False) + else: + monkeypatch.setenv("NEEDS", value) + + with pytest.raises(SystemExit) as refused: + ci_test_plan.main(["require-results"]) + + assert str(refused.value).startswith("NEEDS must hold") + + def test_workflow_job_parser_ignores_nested_yaml_keys() -> None: source = """\ jobs: diff --git a/tools/ci_test_plan.py b/tools/ci_test_plan.py index 3f199c5dcd..c62bb6ccae 100644 --- a/tools/ci_test_plan.py +++ b/tools/ci_test_plan.py @@ -10,7 +10,7 @@ import re import sys import urllib.request -from collections.abc import Callable, Iterable, Mapping +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from dataclasses import dataclass from pathlib import Path, PurePosixPath from typing import Any @@ -87,7 +87,43 @@ class TestGroup: "select-countries": "selects the country-specific test categories", "lint": "runs static checks and verifies this test plan", "wheels": "builds and inspects distribution archives", + "ci": "reports one result for all the other jobs, for the main ruleset to require", } +# ``ci`` gives the ``main`` ruleset one check that covers the country jobs. It +# fails unless every other job passed or the selection left it out. A skipped +# matrix job reports under its unexpanded name (``engine-uk``, not +# ``engine-uk (3.13)``), so a ruleset cannot require ``engine-us`` or +# ``engine-uk`` by name. The ruleset also names both ``engine-free`` legs, and +# that job runs this plan's tests. A change that breaks ``ci`` therefore fails +# a check the ruleset reads without ``ci``. +AGGREGATE_JOB = "ci" +# The job and step whose outputs say which country jobs a run needs. +SELECTION_JOB = "select-countries" +SELECTION_STEP = "plan" +SELECTION_COMMAND = "python3 tools/ci_test_plan.py select-countries" +# The one ``run`` step of ``ci``, and the context GitHub hands it. +REQUIRE_RESULTS_COMMAND = "python3 tools/ci_test_plan.py require-results" +NEEDS_EXPRESSION = "${{ toJSON(needs) }}" +AGGREGATE_JOB_KEYS = ( + "if", + "needs", + "permissions", + "runs-on", + "timeout-minutes", + "steps", +) +# The shell every ``run`` step runs under. GitHub runs an explicit +# ``shell: bash`` as ``bash -eo pipefail``, and a step with no shell as +# ``bash -e``, where a pipe hides the exit status of the command that feeds it. +WORKFLOW_DEFAULTS = {"run": {"shell": "bash"}} +# Step conditions that run a step whenever its job is still passing. +STEP_CONDITIONS = ("always()", "!cancelled()", "success()") +# The tests of this plan, and the job that must run them. The ruleset reads +# that job without ``ci``, so the tests of the code ``ci`` runs stay in it. +PLAN_TESTS = "packages/microcosm-build/tests/engine_free/shared/test_ci_test_plan.py" +PLAN_TESTS_JOB = "engine-free" +# GitHub lists at most this many files of one pull request. +LISTED_FILES_LIMIT = 3000 DOC_PREFIXES = ("docs/",) DOC_FILES = {"LICENSE"} @@ -96,6 +132,78 @@ class TestGroup: RequestJSON = Callable[[str, str], Any] _WORKFLOW_JOB = re.compile(r"^ ([A-Za-z_][A-Za-z0-9_-]*):(?:\s.*)?$") +# Line breaks in YAML 1.1 (PyYAML) that YAML 1.2 reads as ordinary characters. +_YAML_1_1_LINE_BREAKS = "\x85\u2028\u2029" + + +def plan_jobs() -> tuple[str, ...]: + """Return every workflow job the plan declares, except the aggregate job.""" + + return tuple( + name for name in (*WORKFLOW_INFRASTRUCTURE_JOBS, *JOBS) if name != AGGREGATE_JOB + ) + + +def selection_output(job: object) -> str | None: + """Return the selection output that decides whether one job runs. + + A job whose every test group needs the same country's engine runs only + when ``select-countries`` reports that country. Every other job always + runs, and this returns ``None`` for it. + """ + + specs = [spec for spec in TEST_GROUPS.values() if spec.job == job] + countries = {spec.country for spec in specs} + if len(countries) == 1 and all(spec.engine_module for spec in specs): + return f"run_{countries.pop()}" + return None + + +def selection_errors() -> tuple[str, ...]: + """Return errors for jobs that wait on an output the selection never writes.""" + + written = set(select_countries(())) + return tuple( + f"{job}: the selection writes no {output} output" + for job in JOBS + if (output := selection_output(job)) is not None and output not in written + ) + + +def plan_tests_errors() -> tuple[str, ...]: + """Return errors unless ``engine-free`` runs this plan's own tests. + + The selection must not decide that job. The root ``conftest.py`` leaves a + group's files out when the group needs an engine the environment lacks, + or holds integration tests and pytest runs without ``--run-integration``. + """ + + path = ROOT / PLAN_TESTS + group = group_name_for_test_path(path) + spec = TEST_GROUPS.get(group) if group is not None else None + if ( + path in test_files() + and spec is not None + and spec.job == PLAN_TESTS_JOB + and spec.engine_module is None + and not spec.integration + and selection_output(PLAN_TESTS_JOB) is None + ): + return () + return ( + f"{PLAN_TESTS}: {PLAN_TESTS_JOB} must run this file, the plan's own " + "tests, on every pull request, because the main ruleset reads that job " + f"without {AGGREGATE_JOB}. If the tests moved, update PLAN_TESTS", + ) + + +def selection_condition(job: object) -> str | None: + """Return the ``if`` the workflow must give one job, or ``None`` for no ``if``.""" + + output = selection_output(job) + if output is None: + return None + return f"needs['{SELECTION_JOB}'].outputs.{output} == 'true'" def workflow_job_names(source: str) -> tuple[str, ...]: @@ -146,6 +254,340 @@ def workflow_job_errors(source: str) -> tuple[str, ...]: return tuple(errors) +def load_workflow(source: str) -> object: + """Parse workflow YAML, and refuse three things GitHub may read differently. + + PyYAML keeps the last of two keys it reads as one and says nothing, + applies merge keys (``<<``), which GitHub's docs do not list beside anchors + and aliases, and reads U+0085, U+2028 and U+2029 as line breaks, which YAML + 1.2 does not. PyYAML reads two keys as one when it builds the same value + from them, as it does from ``NO`` and ``OFF``. The import sits here because + ``select-countries`` and ``ci`` run this module on bare ``python3``, where + PyYAML may not exist. + """ + + import yaml + + class Loader(yaml.SafeLoader): + pass + + def mapping(loader: yaml.SafeLoader, node: yaml.MappingNode) -> dict: + written: dict[object, str] = {} + for key, _ in node.value: + if key.tag == "tag:yaml.org,2002:merge": + problem = "a mapping uses a merge key (`<<`)" + elif not isinstance(key, yaml.ScalarNode): + problem = "a mapping uses a list or a mapping as a key" + else: + value = loader.construct_object(key) + if value not in written: + written[value] = key.value + continue + problem = ( + f"a mapping sets `{key.value}` twice" + if written[value] == key.value + else f"PyYAML reads `{written[value]}` and `{key.value}` as one key" + ) + raise yaml.constructor.ConstructorError(None, None, problem, key.start_mark) + return loader.construct_mapping(node, deep=True) + + Loader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, mapping) + for number, line in enumerate(source.split("\n"), start=1): + for character in _YAML_1_1_LINE_BREAKS: + if character in line: + raise ValueError( + f"workflow line {number} holds U+{ord(character):04X}, which " + "PyYAML reads as a line break and YAML 1.2 does not" + ) + try: + return yaml.load(source, Loader=Loader) + except yaml.YAMLError as error: + raise ValueError(f"workflow fails to load: {error}") from error + + +def _job_needs(job: Mapping[object, object]) -> tuple[str, ...] | None: + """Return the jobs one job needs. + + Return ``None`` unless ``needs`` holds a job name or a list of them. + """ + + needs = job.get("needs", []) + if isinstance(needs, str): + return (needs,) + if isinstance(needs, list) and all(isinstance(name, str) for name in needs): + return tuple(needs) + return None + + +def _job_steps(job: Mapping[object, object]) -> list[Mapping[object, object]] | None: + """Return one job's steps, or ``None`` unless ``steps`` holds a list of mappings.""" + + steps = job.get("steps", []) + if isinstance(steps, list) and all(isinstance(step, Mapping) for step in steps): + return steps + return None + + +def _steps_within(steps: object) -> Iterator[Mapping[object, object]]: + """Yield every step, and every step that a step groups in a list.""" + + for step in steps if isinstance(steps, list) else (): + if isinstance(step, Mapping): + yield step + for value in step.values(): + yield from _steps_within(value) + + +def _skips_while_passing(step: Mapping[object, object]) -> bool: + """Say whether a step's ``if`` falls outside ``STEP_CONDITIONS``.""" + + condition = step.get("if", "success()") + if not isinstance(condition, str): + return True + text = condition.strip(" ") + if text.startswith("${{") and text.endswith("}}"): + text = text[3:-2].strip(" ") + return text not in STEP_CONDITIONS + + +def _aggregate_errors( + job: Mapping[object, object], + needs: Sequence[str], + steps: Sequence[Mapping[object, object]], + others: Sequence[object], +) -> list[str]: + """Return errors unless ``ci`` always runs, needs every other job, and + does nothing but check out the repository and run ``require-results``.""" + + errors: list[str] = [] + if job.get("if") != "always()": + errors.append( + f"{AGGREGATE_JOB}: `if` must read always(), because GitHub reports a " + "skipped required check as passing" + ) + errors.extend( + f"{AGGREGATE_JOB}: needs must list {name}" + for name in others + if name not in needs + ) + errors.extend( + f"{AGGREGATE_JOB}: needs lists {name}, which the workflow does not declare" + for name in needs + if name not in others + ) + if any(key not in AGGREGATE_JOB_KEYS for key in job): + errors.append( + f"{AGGREGATE_JOB}: job may set only {', '.join(AGGREGATE_JOB_KEYS)}" + ) + checkout, run = steps if len(steps) == 2 else ({}, {}) + if not str(checkout.get("uses")).startswith("actions/checkout@") or any( + key not in ("name", "uses", "with") for key in checkout + ): + errors.append( + f"{AGGREGATE_JOB}: steps must be a checkout that sets only `name`, " + "`uses` and `with`, then one `run` step" + ) + elif ( + run.get("run") != REQUIRE_RESULTS_COMMAND + or run.get("env") != {"NEEDS": NEEDS_EXPRESSION} + or any(key not in ("name", "env", "run") for key in run) + ): + errors.append( + f"{AGGREGATE_JOB}: its `run` step must set `run` to " + f"`{REQUIRE_RESULTS_COMMAND}` on one line, `env` to NEEDS: " + f"{NEEDS_EXPRESSION} alone, and no other key but `name`" + ) + return errors + + +def _selection_job_errors( + job: Mapping[object, object], steps: Sequence[Mapping[object, object]] +) -> list[str]: + """Return errors unless the selection job publishes what its step writes.""" + + errors: list[str] = [] + wanted = { + output: f"${{{{ steps.{SELECTION_STEP}.outputs.{output} }}}}" + for output in select_countries(()) + } + outputs = job.get("outputs") + if not isinstance(outputs, Mapping) or any( + outputs.get(output) != value for output, value in wanted.items() + ): + errors.append( + f"{SELECTION_JOB}: outputs must map each of {', '.join(wanted)} to " + f"`${{{{ steps.{SELECTION_STEP}.outputs. }}}}`" + ) + plans = [step for step in steps if step.get("id") == SELECTION_STEP] + if len(plans) != 1 or plans[0].get("run") != SELECTION_COMMAND: + errors.append( + f"{SELECTION_JOB}: one step with id `{SELECTION_STEP}` must run " + f"`{SELECTION_COMMAND}` on one line" + ) + return errors + + +def workflow_document_errors(document: object) -> tuple[str, ...]: + """Return an error for each way the parsed workflow breaks the test plan. + + ``document`` holds the workflow as a YAML parser reads it. These rules keep + a failure from passing. The plan and the workflow declare the same jobs, + no job or step sets ``continue-on-error``, no step can skip while its job + passes, every ``run`` step runs under ``bash -eo pipefail``, and no job sets + ``name``. ``ci`` runs whatever the other jobs report, needs each of them, + and does nothing but hand their results to ``require-results``. The + selection job publishes the outputs its ``plan`` step writes. Two more + rules keep a run from failing for no fault of its + pull request. Each other job carries the ``if`` the plan gives it, and + needs ``select-countries`` when that ``if`` reads its outputs. A job needs + only jobs that run whenever it runs. + """ + + jobs = document.get("jobs") if isinstance(document, Mapping) else None + if not isinstance(jobs, Mapping): + return ("workflow has no jobs mapping",) + expected = (*plan_jobs(), AGGREGATE_JOB) + errors = [ + f"{name}: the test plan doesn't declare this workflow job" + for name in jobs + if name not in expected + ] + if document.get("defaults") != WORKFLOW_DEFAULTS: + errors.append( + "workflow: `defaults` must set `run: shell: bash` and nothing else, " + "so every `run` step runs under `bash -eo pipefail` and a pipe cannot " + "hide a failed command" + ) + errors.extend( + f"{name}: the workflow lacks this job of the test plan" + for name in expected + if name not in jobs + ) + for name, job in jobs.items(): + if not isinstance(job, Mapping): + errors.append(f"{name}: job must be a mapping") + continue + needs, steps = _job_needs(job), _job_steps(job) + if needs is None: + errors.append(f"{name}: needs must be a job name or a list of job names") + needs = () + if steps is None: + errors.append(f"{name}: steps must be a list of mappings") + steps = [] + if name == AGGREGATE_JOB: + others = [other for other in jobs if other != AGGREGATE_JOB] + errors.extend(_aggregate_errors(job, needs, steps, others)) + continue + if "continue-on-error" in job: + errors.append( + f"{name}: job sets `continue-on-error`, which can let the run " + "pass when the job fails" + ) + if "name" in job: + errors.append( + f"{name}: job sets `name`, which would report the job under " + "another check name" + ) + within = tuple(_steps_within(steps)) + if any("continue-on-error" in step for step in within): + errors.append( + f"{name}: a step sets `continue-on-error`, which can let the job " + "pass when the step fails" + ) + if any(map(_skips_while_passing, within)): + errors.append( + f"{name}: a step sets an `if` other than `always()`, `success()` " + "or `!cancelled()`. Write `if: always()`, `if: ${{ !cancelled() }}` " + "or no `if`, so the step runs whenever its job passes" + ) + if "defaults" in job or any("shell" in step for step in within): + errors.append( + f"{name}: job sets `defaults` or a step sets `shell`, which " + "can replace the workflow's `bash -eo pipefail`" + ) + if name == SELECTION_JOB: + errors.extend(_selection_job_errors(job, steps)) + condition, output = selection_condition(name), selection_output(name) + if condition is None and "if" in job: + errors.append( + f"{name}: job must have no `if`, because the plan runs it on " + f"every run and {AGGREGATE_JOB} refuses its skip" + ) + elif condition is not None and job.get("if") != condition: + errors.append(f"{name}: `if` must read {condition}") + elif condition is not None and SELECTION_JOB not in needs: + errors.append(f"{name}: needs must list {SELECTION_JOB}") + errors.extend( + f"{name}: needs {needed}, which the selection may leave out when " + f"it runs {name}, and GitHub would then skip {name} too" + for needed in needs + if selection_output(needed) not in (None, output) + ) + return tuple(errors) + + +def _job_result(needs: Mapping[str, object], job: str) -> object: + entry = needs.get(job) + return entry.get("result") if isinstance(entry, Mapping) else None + + +def _unplanned_skip(needs: Mapping[str, object], output: str | None) -> str | None: + """Say why the selection did not ask for this skip, or ``None`` when it did.""" + + if output is None: + return "but the plan runs it on every run" + if _job_result(needs, SELECTION_JOB) != "success": + return f"and {SELECTION_JOB} did not succeed" + entry = needs.get(SELECTION_JOB) + outputs = entry.get("outputs") if isinstance(entry, Mapping) else None + selected = outputs.get(output) if isinstance(outputs, Mapping) else None + if selected == "false": + return None + if selected == "true": + return f"but {SELECTION_JOB} selected it ({output}=true)" + return f"and {SELECTION_JOB} did not report {output} as true or false" + + +def aggregate_result_errors( + needs: Mapping[str, object], + *, + jobs: Sequence[str], + selected_by: Mapping[str, str], +) -> tuple[str, ...]: + """Return errors unless every job passed, or the selection left it out. + + ``needs`` holds GitHub's ``needs`` context for the aggregate job: one + entry per needed job with its ``result`` (``success``, ``failure``, + ``cancelled`` or ``skipped``) and its ``outputs``. ``selected_by`` names + the selection output that decides each job the selection may leave out. A + skip passes only when ``selected_by`` names the job, the selection job + succeeded and that output reads ``false``. Any other result, a missing job + and a job outside ``jobs`` each add an error, so the aggregate job passes + only when every job in ``jobs`` reported, none failed, and nothing skipped + that the selection asked for. + """ + + errors: list[str] = [] + for job in jobs: + result = _job_result(needs, job) + if result == "success": + continue + if result is None: + errors.append(f"{job}: reported no result") + elif result != "skipped": + errors.append(f"{job}: {result}") + else: + reason = _unplanned_skip(needs, selected_by.get(job)) + if reason is not None: + errors.append(f"{job}: skipped, {reason}") + errors.extend( + f"{job}: reported a result, but the plan declares no such job" + for job in needs + if job not in jobs + ) + return tuple(errors) + + def engine_guard_lines(source: str) -> tuple[int, ...]: """Return lines containing country-engine availability checks.""" @@ -302,6 +744,8 @@ def verify() -> None: raise SystemExit("no test files found below packages/*/tests") errors = list(workflow_job_errors(WORKFLOW.read_text(encoding="utf-8"))) + errors.extend(selection_errors()) + errors.extend(plan_tests_errors()) counts = {name: 0 for name in TEST_GROUPS} seen_directories: dict[tuple[str, str], str] = {} seen_report_categories: dict[tuple[str, str], str] = {} @@ -428,10 +872,16 @@ def pull_request_paths( *, request_json: RequestJSON = api_json, ) -> list[str]: - """Return every current and previous path from a pull request diff.""" + """Return every current and previous path from a pull request diff. + + GitHub lists at most ``LISTED_FILES_LIMIT`` files. A listing that reaches + the limit may leave files out, so this returns an empty list for it, and + an empty list selects both countries. + """ number = event["pull_request"]["number"] paths: list[str] = [] + listed = 0 page = 1 while True: files = request_json( @@ -441,6 +891,9 @@ def pull_request_paths( ) if not isinstance(files, list): raise TypeError("pull request files response must be a list") + listed += len(files) + if listed >= LISTED_FILES_LIMIT: + return [] paths.extend(_file_paths(files)) if len(files) < 100: return paths @@ -486,24 +939,72 @@ def write_country_selection() -> None: for path in paths: print(path) else: - print("pull request has no changed paths: all country jobs selected") + print( + "pull request lists no changed paths, or GitHub's limit of " + f"{LISTED_FILES_LIMIT:,} files: all country jobs selected" + ) print("country selection:", selected) +def verify_workflow() -> None: + """Fail unless the workflow loads and ``workflow_document_errors`` finds nothing.""" + + try: + document = load_workflow(WORKFLOW.read_text(encoding="utf-8")) + except ImportError as error: + raise SystemExit( + "verify-workflow needs PyYAML: run " + "`uv run python tools/ci_test_plan.py verify-workflow`" + ) from error + except ValueError as error: + raise SystemExit(str(error)) from error + errors = workflow_document_errors(document) + if errors: + raise SystemExit("workflow fails the test plan:\n " + "\n ".join(errors)) + print("workflow=ok") + + +def require_results() -> None: + """Fail unless ``NEEDS`` shows each job passed, or the selection left it out.""" + + jobs = plan_jobs() + selected_by = { + job: output for job in jobs if (output := selection_output(job)) is not None + } + try: + needs = json.loads(os.environ["NEEDS"]) + except (KeyError, json.JSONDecodeError) as error: + raise SystemExit( + f"NEEDS must hold the workflow's `needs` context as JSON: {error!r}" + ) from error + if not isinstance(needs, Mapping): + raise SystemExit("NEEDS must hold a JSON object, one entry per needed job") + for job in jobs: + print(f"{job}={_job_result(needs, job)}", flush=True) + errors = aggregate_result_errors(needs, jobs=jobs, selected_by=selected_by) + if errors: + raise SystemExit("workflow did not pass:\n " + "\n ".join(errors)) + print("results=ok", flush=True) + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) commands = parser.add_subparsers(dest="command", required=True) commands.add_parser("verify") + commands.add_parser("verify-workflow") commands.add_parser("describe") list_parser = commands.add_parser("list") list_parser.add_argument("group", choices=tuple(TEST_GROUPS)) list_job_parser = commands.add_parser("list-job") list_job_parser.add_argument("job", choices=JOBS) commands.add_parser("select-countries") + commands.add_parser("require-results") args = parser.parse_args(argv) if args.command == "verify": verify() + elif args.command == "verify-workflow": + verify_workflow() elif args.command == "describe": print(describe(), end="") elif args.command == "list": @@ -514,6 +1015,8 @@ def main(argv: list[str] | None = None) -> int: print(path.relative_to(ROOT)) elif args.command == "select-countries": write_country_selection() + elif args.command == "require-results": + require_results() return 0