From 2b9f838b5fbcae86428dde5f3c62e47c05af002f Mon Sep 17 00:00:00 2001 From: Payton McIntosh Date: Sun, 31 May 2026 19:46:24 +0100 Subject: [PATCH 01/32] Add rendered workflow contracts for Python template Render and assert both pure-Python and Python/Rust template variants using Rust-template-style helpers. Parse generated TOML and workflow YAML so the pytest-copier suite checks the local gates, coverage action inputs, release wheel jobs, and checkout credential policy. Template `release.yml` as Jinja so Rust projects use the wheel matrix while pure-Python projects keep the single pure-wheel path. --- scripts/setup_test_deps.sh | 2 +- .../.github/actions/build-wheels/action.yml | 1 + .../actions/pure-python-wheel/action.yml | 1 + template/.github/workflows/build-wheels.yml | 2 + template/.github/workflows/ci.yml.jinja | 2 + .../{release.yml => release.yml.jinja} | 21 +- tests/__init__.py | 1 + tests/__snapshots__/test_template.ambr | 1 + tests/test_github_actions_integration.py | 27 +- tests/test_template.py | 519 ++++++++++++++++-- 10 files changed, 525 insertions(+), 52 deletions(-) rename template/.github/workflows/{release.yml => release.yml.jinja} (67%) create mode 100644 tests/__init__.py diff --git a/scripts/setup_test_deps.sh b/scripts/setup_test_deps.sh index eef3164..de84522 100755 --- a/scripts/setup_test_deps.sh +++ b/scripts/setup_test_deps.sh @@ -3,4 +3,4 @@ # Generated projects install their own linting and type-checking dependencies. set -euo pipefail -pip install pytest-copier syrupy +pip install pytest-copier PyYAML syrupy diff --git a/template/.github/actions/build-wheels/action.yml b/template/.github/actions/build-wheels/action.yml index 85680ed..1b0fc51 100644 --- a/template/.github/actions/build-wheels/action.yml +++ b/template/.github/actions/build-wheels/action.yml @@ -16,6 +16,7 @@ runs: - uses: actions/checkout@v4 with: fetch-depth: 0 + persist-credentials: false - name: Set up Python uses: actions/setup-python@v5 with: diff --git a/template/.github/actions/pure-python-wheel/action.yml b/template/.github/actions/pure-python-wheel/action.yml index 656e791..2eeedd9 100644 --- a/template/.github/actions/pure-python-wheel/action.yml +++ b/template/.github/actions/pure-python-wheel/action.yml @@ -17,6 +17,7 @@ runs: - uses: actions/checkout@v4 with: fetch-depth: 0 + persist-credentials: false - name: Set up Python uses: actions/setup-python@v5 with: diff --git a/template/.github/workflows/build-wheels.yml b/template/.github/workflows/build-wheels.yml index c83b9df..822fd13 100644 --- a/template/.github/workflows/build-wheels.yml +++ b/template/.github/workflows/build-wheels.yml @@ -34,6 +34,8 @@ jobs: cibw_arch: arm64 steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: ./.github/actions/build-wheels with: python-version: ${{ inputs['python-version'] }} diff --git a/template/.github/workflows/ci.yml.jinja b/template/.github/workflows/ci.yml.jinja index dbb218a..6c751ea 100644 --- a/template/.github/workflows/ci.yml.jinja +++ b/template/.github/workflows/ci.yml.jinja @@ -19,6 +19,8 @@ jobs: steps: - name: Check out repository uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + with: + persist-credentials: false - name: Set up Python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 diff --git a/template/.github/workflows/release.yml b/template/.github/workflows/release.yml.jinja similarity index 67% rename from template/.github/workflows/release.yml rename to template/.github/workflows/release.yml.jinja index 0cb837d..4b47466 100644 --- a/template/.github/workflows/release.yml +++ b/template/.github/workflows/release.yml.jinja @@ -6,7 +6,7 @@ on: - 'v*.*.*' concurrency: - group: release-${{ github.ref }} + group: release-${{ "{{" }} github.ref {{ "}}" }} cancel-in-progress: true permissions: @@ -15,6 +15,12 @@ permissions: jobs: +{% if use_rust %} + build-wheels: + uses: ./.github/workflows/build-wheels.yml + with: + python-version: '3.13' +{% else %} pure-wheel: runs-on: ubuntu-latest steps: @@ -22,21 +28,30 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 + persist-credentials: false - uses: ./.github/actions/pure-python-wheel with: python-version: '3.13' artifact-name: wheels-pure +{% endif %} release: +{% if not use_rust %} # This project has no C or Rust extensions, so cross-platform # builds are unnecessary. Only the pure Python wheel is published. +{% endif %} needs: +{% if use_rust %} + - build-wheels +{% else %} - pure-wheel +{% endif %} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: fetch-depth: 0 + persist-credentials: false - uses: softprops/action-gh-release@v2 with: generate_release_notes: true @@ -47,6 +62,6 @@ jobs: run: | set -eu find dist/wheels-* -type f -name "*.whl" -print0 | \ - xargs -0 -r gh release upload "${{ github.ref_name }}" + xargs -0 -r gh release upload "${{ "{{" }} github.ref_name {{ "}}" }}" env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ "{{" }} secrets.GITHUB_TOKEN {{ "}}" }} diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..7f1a999 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""Test support package for the Copier template validation suite.""" diff --git a/tests/__snapshots__/test_template.ambr b/tests/__snapshots__/test_template.ambr index 3c4f4c2..3224ee8 100644 --- a/tests/__snapshots__/test_template.ambr +++ b/tests/__snapshots__/test_template.ambr @@ -57,5 +57,6 @@ nixie Validate Mermaid diagrams test Run tests help Show available targets + ''' # --- diff --git a/tests/test_github_actions_integration.py b/tests/test_github_actions_integration.py index e68509d..ee22d0c 100644 --- a/tests/test_github_actions_integration.py +++ b/tests/test_github_actions_integration.py @@ -36,6 +36,7 @@ ACT_IMAGE = "ubuntu-latest=catthehacker/ubuntu:act-latest" GENERATE_COVERAGE_STEP = "Test and Measure Coverage" + def prepare_git_repository(project: CopierProject) -> None: """Initialise a rendered project as a Git repository for act. @@ -234,12 +235,8 @@ def assert_ci_exercised_expected_steps(logs: str, *, use_rust: bool) -> None: saw_python = False saw_rust = not use_rust for event in iter_json_log_events(logs): - output = str( - event_text(event, "Output", "output", "message", "msg") - ) - step = str( - event_text(event, "name", "step_name", "Step", "step") - ) + output = str(event_text(event, "Output", "output", "message", "msg")) + step = str(event_text(event, "name", "step_name", "Step", "step")) in_coverage_step = GENERATE_COVERAGE_STEP in step saw_coverage = saw_coverage or ( in_coverage_step @@ -296,9 +293,9 @@ def assert_act_result( assert_act_result(project, code, logs, use_rust=False) """ - assert ( - project / "coverage.xml" - ).exists(), "act workflow should write coverage.xml in the generated project" + assert (project / "coverage.xml").exists(), ( + "act workflow should write coverage.xml in the generated project" + ) assert_ci_exercised_expected_steps(logs, use_rust=use_rust) if code == 0: return @@ -380,11 +377,11 @@ def test_generated_workflow_runs_with_shared_coverage_action( workflow = (project / ".github" / "workflows" / "ci.yml").read_text( encoding="utf-8" ) - assert ( - "leynos/shared-actions/.github/actions/generate-coverage" in workflow - ), "Generated workflow should use the shared generate-coverage action" + assert "leynos/shared-actions/.github/actions/generate-coverage" in workflow, ( + "Generated workflow should use the shared generate-coverage action" + ) if use_rust: - assert ( - "cargo-manifest: rust_extension/Cargo.toml" in workflow - ), "Rust workflow should pass the Rust extension manifest to coverage" + assert "cargo-manifest: rust_extension/Cargo.toml" in workflow, ( + "Rust workflow should pass the Rust extension manifest to coverage" + ) assert_act_result(project, code, logs, use_rust=use_rust) diff --git a/tests/test_template.py b/tests/test_template.py index c87cf92..e12b38a 100644 --- a/tests/test_template.py +++ b/tests/test_template.py @@ -23,9 +23,12 @@ from __future__ import annotations import shlex +import tomllib from pathlib import Path +from typing import Any import pytest +import yaml from pytest_copier.plugin import CopierFixture, CopierProject from syrupy.assertion import SnapshotAssertion @@ -60,6 +63,46 @@ def run_quality_gates(project: CopierProject) -> None: project.run("make all") +def render_project( + tmp_path: Path, + copier: CopierFixture, + *, + project_name: str, + package_name: str, + use_rust: bool = False, + python_version: str = "3.10", +) -> CopierProject: + """Render a generated Python project with explicit template answers. + + Parameters + ---------- + tmp_path + Temporary directory used as the generated project destination. + copier + ``pytest-copier`` fixture bound to this template repository. + project_name + Project name answer passed to Copier. + package_name + Package name answer passed to Copier. + use_rust + Whether to include the optional PyO3 extension. + python_version + Minimum supported Python version answer passed to Copier. + + Returns + ------- + CopierProject + Rendered project wrapper for file assertions and command execution. + """ + return copier.copy( + tmp_path, + project_name=project_name, + package_name=package_name, + use_rust=use_rust, + python_version=python_version, + ) + + def check_generated_import(project: CopierProject, package: str, greeting: str) -> None: """Import a generated package and assert its greeting. @@ -125,6 +168,51 @@ def read_generated_file(project: CopierProject, relative_path: str) -> str: return (project / relative_path).read_text(encoding="utf-8") +def read_generated_text(path: Path) -> str: + """Read a generated file with assertion-focused error context.""" + try: + return path.read_text(encoding="utf-8") + except OSError as error: + pytest.fail(f"could not read generated file {path}: {error}") + + +def parse_toml_file(path: Path) -> dict[str, Any]: + """Parse generated TOML with assertion-focused error context.""" + text = read_generated_text(path) + try: + parsed = tomllib.loads(text) + except tomllib.TOMLDecodeError as error: + pytest.fail(f"could not parse generated TOML {path}: {error}") + return parsed + + +def parse_yaml_mapping(text: str, label: str) -> dict[str, Any]: + """Parse generated YAML as a mapping with clear failure context.""" + try: + parsed = yaml.safe_load(text) + except yaml.YAMLError as error: + pytest.fail(f"could not parse generated {label}: {error}") + if not isinstance(parsed, dict): + pytest.fail(f"expected generated {label} to parse as a mapping") + return parsed + + +def require_mapping(mapping: dict[str, Any], key: str, label: str) -> dict[str, Any]: + """Return a nested mapping or fail with the missing schema path.""" + value = mapping.get(key) + if not isinstance(value, dict): + pytest.fail(f"expected {label} to include mapping key {key!r}") + return value + + +def require_sequence(mapping: dict[str, Any], key: str, label: str) -> list[Any]: + """Return a nested sequence or fail with the missing schema path.""" + value = mapping.get(key) + if not isinstance(value, list): + pytest.fail(f"expected {label} to include sequence key {key!r}") + return value + + def assert_common_make_targets(makefile: str) -> None: """Assert Makefile targets shared by all generated variants. @@ -274,20 +362,20 @@ def test_python_only_template(copier: CopierFixture, tmp_path: Path) -> None: ) run_quality_gates(proj) - assert not ( - proj / "rust_extension" - ).exists(), "rust_extension directory should not exist for Python-only template" - assert not ( - proj / "docs" / "rust-extension.md" - ).exists(), "Rust documentation should not be generated for Python-only template" - assert ( - "maturin" not in (proj / "pyproject.toml").read_text(encoding="utf-8") - ), "maturin should not be in pyproject.toml for Python-only template" + assert not (proj / "rust_extension").exists(), ( + "rust_extension directory should not exist for Python-only template" + ) + assert not (proj / "docs" / "rust-extension.md").exists(), ( + "Rust documentation should not be generated for Python-only template" + ) + assert "maturin" not in (proj / "pyproject.toml").read_text(encoding="utf-8"), ( + "maturin should not be in pyproject.toml for Python-only template" + ) makefile = read_generated_file(proj, "Makefile") assert_common_make_targets(makefile) - assert ( - "lint-rust" not in makefile - ), "Python-only Makefile should not expose lint-rust" + assert "lint-rust" not in makefile, ( + "Python-only Makefile should not expose lint-rust" + ) check_generated_import(proj, "pure_pkg", "hello from Python") @@ -322,23 +410,23 @@ def test_rust_template(copier: CopierFixture, tmp_path: Path) -> None: ) run_quality_gates(proj) - assert ( - proj / "rust_extension" - ).exists(), "rust_extension directory should exist for Rust template" - assert ( - proj / "docs" / "rust-extension.md" - ).exists(), "Rust documentation should be generated for Rust template" - assert ( - "maturin" in (proj / "pyproject.toml").read_text(encoding="utf-8") - ), "maturin should be in pyproject.toml for Rust template" + assert (proj / "rust_extension").exists(), ( + "rust_extension directory should exist for Rust template" + ) + assert (proj / "docs" / "rust-extension.md").exists(), ( + "Rust documentation should be generated for Rust template" + ) + assert "maturin" in (proj / "pyproject.toml").read_text(encoding="utf-8"), ( + "maturin should be in pyproject.toml for Rust template" + ) makefile = read_generated_file(proj, "Makefile") assert_common_make_targets(makefile) - assert ( - "lint-rust: build whitaker" in makefile - ), "Rust Makefile should expose lint-rust" - assert ( - "cargo is required for Rust tests" in makefile - ), "Rust Makefile should fail clearly when cargo is unavailable" + assert "lint-rust: build whitaker" in makefile, ( + "Rust Makefile should expose lint-rust" + ) + assert "cargo is required for Rust tests" in makefile, ( + "Rust Makefile should fail clearly when cargo is unavailable" + ) check_generated_import(proj, "rust_pkg", "hello from Rust") @@ -373,12 +461,377 @@ def test_rust_template_custom_package(copier: CopierFixture, tmp_path: Path) -> ) run_quality_gates(proj) - assert ( - proj / "rust_extension" - ).exists(), "rust_extension directory should exist for custom package Rust template" + assert (proj / "rust_extension").exists(), ( + "rust_extension directory should exist for custom package Rust template" + ) text = (proj / "pyproject.toml").read_text(encoding="utf-8") - assert ( - "custom_pkg" in text - ), "custom package name should appear in pyproject.toml" + assert "custom_pkg" in text, "custom package name should appear in pyproject.toml" check_generated_import(proj, "custom_pkg", "hello from Rust") + + +@pytest.mark.parametrize( + ("target_dir", "project_name", "package_name", "use_rust"), + [ + ("tooling-pure", "ToolingPure", "tooling_pure", False), + ("tooling-rust", "ToolingRust", "tooling_rust", True), + ], +) +def test_generated_tooling_contracts( + copier: CopierFixture, + tmp_path: Path, + target_dir: str, + project_name: str, + package_name: str, + use_rust: bool, +) -> None: + """Generated variants expose the expected Python and optional Rust tooling.""" + project = render_project( + tmp_path / target_dir, + copier, + project_name=project_name, + package_name=package_name, + use_rust=use_rust, + ) + + run_quality_gates(project) + project.run("uv tool run mbake validate Makefile") + + pyproject = parse_toml_file(project / "pyproject.toml") + makefile = read_generated_text(project / "Makefile") + ci_workflow = read_generated_text(project / ".github" / "workflows" / "ci.yml") + release_workflow = read_generated_text( + project / ".github" / "workflows" / "release.yml" + ) + build_wheels_workflow = read_generated_text( + project / ".github" / "workflows" / "build-wheels.yml" + ) + build_wheels_action = read_generated_text( + project / ".github" / "actions" / "build-wheels" / "action.yml" + ) + pure_wheel_action = read_generated_text( + project / ".github" / "actions" / "pure-python-wheel" / "action.yml" + ) + + assert_generated_tooling_contracts( + package_name=package_name, + pyproject=pyproject, + makefile=makefile, + ci_workflow=ci_workflow, + release_workflow=release_workflow, + build_wheels_workflow=build_wheels_workflow, + build_wheels_action=build_wheels_action, + pure_wheel_action=pure_wheel_action, + use_rust=use_rust, + ) + + +@pytest.mark.parametrize( + ("target_dir", "project_name", "package_name", "use_rust"), + [ + ("workflow-pure", "WorkflowPure", "workflow_pure", False), + ("workflow-rust", "WorkflowRust", "workflow_rust", True), + ], +) +def test_generated_github_workflows_match_act_validation_contract( + copier: CopierFixture, + tmp_path: Path, + target_dir: str, + project_name: str, + package_name: str, + use_rust: bool, +) -> None: + """Rendered workflows expose stable black-box inputs for act validation.""" + project = render_project( + tmp_path / target_dir, + copier, + project_name=project_name, + package_name=package_name, + use_rust=use_rust, + ) + ci_workflow = read_generated_text(project / ".github" / "workflows" / "ci.yml") + parsed_ci_workflow = parse_yaml_mapping(ci_workflow, "CI workflow") + jobs = require_mapping(parsed_ci_workflow, "jobs", "CI workflow") + lint_test = require_mapping(jobs, "lint-test", "CI workflow jobs") + steps = require_sequence(lint_test, "steps", "CI lint-test job") + + assert checkout_steps_disable_credentials(steps), ( + "expected CI checkout steps to disable credential persistence" + ) + coverage_steps = [ + step + for step in steps + if isinstance(step, dict) and step.get("name") == "Test and Measure Coverage" + ] + assert len(coverage_steps) == 1, "expected one shared coverage action step" + coverage_step = coverage_steps[0] + assert ( + coverage_step.get("uses") + == "leynos/shared-actions/.github/actions/generate-coverage" + "@d400b079fb6a8fa92f7e7b6c57f3d1c92a4b2d54" + ), "expected CI to use the pinned shared coverage action" + coverage_inputs = require_mapping(coverage_step, "with", "coverage step") + assert coverage_inputs.get("output-path") == "coverage.xml", ( + "expected CI coverage output path to match the act assertion" + ) + assert coverage_inputs.get("format") == "cobertura", ( + "expected CI coverage format to match the CodeScene upload" + ) + assert coverage_inputs.get("artefact-name-suffix") == package_name.replace( + "_", "-" + ), "expected package-specific coverage artefact name suffix" + if use_rust: + assert coverage_inputs.get("cargo-manifest") == "rust_extension/Cargo.toml", ( + "expected Rust variant to pass the extension manifest to coverage" + ) + else: + assert "cargo-manifest" not in coverage_inputs, ( + "expected pure-Python variant to omit Rust coverage inputs" + ) + + +def assert_generated_tooling_contracts( + *, + package_name: str, + pyproject: dict[str, Any], + makefile: str, + ci_workflow: str, + release_workflow: str, + build_wheels_workflow: str, + build_wheels_action: str, + pure_wheel_action: str, + use_rust: bool, +) -> None: + """Assert generated Python/Rust tooling contracts from one validator.""" + assert_pyproject_contracts( + package_name=package_name, + pyproject=pyproject, + use_rust=use_rust, + ) + assert_makefile_contracts(makefile=makefile, use_rust=use_rust) + assert_ci_workflow_contracts(ci_workflow=ci_workflow, use_rust=use_rust) + assert_release_workflow_contracts( + release_workflow=release_workflow, + use_rust=use_rust, + ) + assert_wheel_workflow_contracts( + build_wheels_workflow=build_wheels_workflow, + build_wheels_action=build_wheels_action, + pure_wheel_action=pure_wheel_action, + ) + + +def assert_pyproject_contracts( + *, package_name: str, pyproject: dict[str, Any], use_rust: bool +) -> None: + """Assert generated Python packaging contracts.""" + project = require_mapping(pyproject, "project", "pyproject.toml") + assert project.get("name"), "expected generated project metadata to include a name" + assert project.get("requires-python") == ">=3.10", ( + "expected generated pyproject.toml to use the requested Python version" + ) + dependency_groups = require_mapping( + pyproject, + "dependency-groups", + "pyproject.toml", + ) + dev_dependencies = dependency_groups.get("dev") + assert isinstance(dev_dependencies, list), ( + "expected generated pyproject.toml to include a dev dependency group" + ) + for dependency in ["pytest", "ruff", "pyright", "ty", "pytest-xdist"]: + assert dependency in dev_dependencies, ( + f"expected generated dev dependencies to include {dependency}" + ) + + build_system = require_mapping(pyproject, "build-system", "pyproject.toml") + if use_rust: + assert build_system.get("build-backend") == "maturin", ( + "expected Rust variant to use maturin as the build backend" + ) + maturin = require_mapping(pyproject, "tool", "pyproject.toml").get("maturin") + assert isinstance(maturin, dict), ( + "expected Rust variant to include tool.maturin configuration" + ) + assert maturin.get("manifest-path") == "rust_extension/Cargo.toml", ( + "expected Rust variant to point maturin at the extension manifest" + ) + assert maturin.get("module-name") == f"{package_name}._{package_name}_rs", ( + "expected Rust variant to render the package-specific module name" + ) + else: + assert build_system.get("build-backend") == "hatchling.build", ( + "expected pure-Python variant to use hatchling as the build backend" + ) + assert "maturin" not in pyproject.get("tool", {}), ( + "expected pure-Python variant to omit maturin configuration" + ) + + +def assert_makefile_contracts(*, makefile: str, use_rust: bool) -> None: + """Assert generated Makefile contracts for both template variants.""" + assert_common_make_targets(makefile) + assert "PYTHON_TARGETS ?=" in makefile, ( + "expected generated Makefile to define Python target selection" + ) + assert "PYLINT_PYPY_SHIM_REF ?=" in makefile, ( + "expected generated Makefile to expose the Pylint shim revision" + ) + assert "test: build $(VENV_TOOLS)" in makefile, ( + "expected generated Makefile test target to depend on the project env" + ) + if use_rust: + assert "TEST_CMD :=" in makefile, ( + "expected Rust variant to select nextest or cargo test" + ) + assert "lint-rust: build whitaker" in makefile, ( + "expected Rust variant to expose the Rust lint target" + ) + assert "cargo is required for Rust tests" in makefile, ( + "expected Rust variant to fail clearly without cargo" + ) + assert "$(CARGO) $(TEST_CMD) $(TEST_FLAGS)" in makefile, ( + "expected Rust variant tests to use the selected cargo test command" + ) + else: + assert "lint-rust" not in makefile, ( + "expected pure-Python variant to omit Rust lint targets" + ) + assert "TEST_CMD :=" not in makefile, ( + "expected pure-Python variant to omit Rust test command selection" + ) + + +def assert_ci_workflow_contracts(*, ci_workflow: str, use_rust: bool) -> None: + """Assert generated CI workflow contracts.""" + parsed_ci_workflow = parse_yaml_mapping(ci_workflow, "CI workflow") + jobs = require_mapping(parsed_ci_workflow, "jobs", "CI workflow") + lint_test = require_mapping(jobs, "lint-test", "CI workflow jobs") + steps = require_sequence(lint_test, "steps", "CI lint-test job") + assert checkout_steps_disable_credentials(steps), ( + "expected generated CI workflow to disable checkout credentials" + ) + assert "make check-fmt" in ci_workflow, ( + "expected generated CI workflow to run the formatting gate" + ) + assert "make lint" in ci_workflow, ( + "expected generated CI workflow to run the lint gate" + ) + assert "make typecheck" in ci_workflow, ( + "expected generated CI workflow to run the typecheck gate" + ) + assert "make build" in ci_workflow, ( + "expected generated CI workflow to build the project before checks" + ) + assert "leynos/shared-actions/.github/actions/generate-coverage" in ci_workflow, ( + "expected generated CI workflow to use the shared coverage action" + ) + if use_rust: + assert "leynos/shared-actions/.github/actions/setup-rust" in ci_workflow, ( + "expected Rust variant CI to set up Rust" + ) + assert "Cache Rust lint and test tools" in ci_workflow, ( + "expected Rust variant CI to cache Rust tools" + ) + assert "cargo-manifest: rust_extension/Cargo.toml" in ci_workflow, ( + "expected Rust variant CI to pass the Rust manifest to coverage" + ) + else: + assert "setup-rust" not in ci_workflow, ( + "expected pure-Python CI to omit Rust setup" + ) + assert "cargo-manifest" not in ci_workflow, ( + "expected pure-Python CI to omit Rust coverage inputs" + ) + + +def assert_release_workflow_contracts(*, release_workflow: str, use_rust: bool) -> None: + """Assert generated release workflow contracts.""" + parsed_release_workflow = parse_yaml_mapping(release_workflow, "release workflow") + jobs = require_mapping(parsed_release_workflow, "jobs", "release workflow") + release = require_mapping(jobs, "release", "release workflow jobs") + release_steps = require_sequence(release, "steps", "release job") + assert checkout_steps_disable_credentials(release_steps), ( + "expected generated release workflow to disable checkout credentials" + ) + assert "softprops/action-gh-release@v2" in release_workflow, ( + "expected release workflow to create a GitHub release" + ) + assert "actions/download-artifact@v4" in release_workflow, ( + "expected release workflow to download wheel artefacts" + ) + if use_rust: + assert "build-wheels" in jobs, ( + "expected Rust variant release workflow to build platform wheels" + ) + build_wheels = require_mapping(jobs, "build-wheels", "release workflow jobs") + assert build_wheels.get("uses") == "./.github/workflows/build-wheels.yml", ( + "expected Rust variant release workflow to call build-wheels.yml" + ) + assert release.get("needs") == ["build-wheels"], ( + "expected Rust variant release job to wait for platform wheels" + ) + assert "pure-wheel" not in jobs, ( + "expected Rust variant release workflow to skip pure wheel job" + ) + else: + assert "pure-wheel" in jobs, ( + "expected pure-Python release workflow to build one pure wheel" + ) + assert release.get("needs") == ["pure-wheel"], ( + "expected pure-Python release job to wait for the pure wheel" + ) + assert "build-wheels" not in jobs, ( + "expected pure-Python release workflow to skip platform wheel matrix" + ) + + +def assert_wheel_workflow_contracts( + *, + build_wheels_workflow: str, + build_wheels_action: str, + pure_wheel_action: str, +) -> None: + """Assert generated wheel workflow and action contracts.""" + parsed_build_wheels = parse_yaml_mapping( + build_wheels_workflow, + "build-wheels workflow", + ) + build_jobs = require_mapping(parsed_build_wheels, "jobs", "build-wheels workflow") + build_job = require_mapping(build_jobs, "build", "build-wheels workflow jobs") + strategy = require_mapping(build_job, "strategy", "build-wheels build job") + matrix = require_mapping(strategy, "matrix", "build-wheels strategy") + includes = require_sequence(matrix, "include", "build-wheels matrix") + assert len(includes) == 6, ( + "expected build-wheels workflow to cover Linux, Windows, and macOS" + ) + assert "persist-credentials: false" in build_wheels_workflow, ( + "expected build-wheels workflow checkout to disable credentials" + ) + assert "uvx --with 'cibuildwheel>=2.16.0,<4.0.0' cibuildwheel" in ( + build_wheels_action + ), "expected Rust wheel action to build through cibuildwheel" + assert "persist-credentials: false" in build_wheels_action, ( + "expected build-wheels action checkout to disable credentials" + ) + assert "uv build --wheel" in pure_wheel_action, ( + "expected pure-wheel action to build through uv" + ) + assert "persist-credentials: false" in pure_wheel_action, ( + "expected pure-wheel action checkout to disable credentials" + ) + + +def checkout_steps_disable_credentials(steps: list[Any]) -> bool: + """Return whether all checkout steps disable credential persistence.""" + checkout_steps = [ + step + for step in steps + if isinstance(step, dict) + and str(step.get("uses", "")).startswith("actions/checkout@") + ] + return bool(checkout_steps) and all( + isinstance(step.get("with"), dict) + and step["with"].get("persist-credentials") is False + for step in checkout_steps + ) From 6da5178803bbf71f3b1c0e4cb872b9d03e85a116 Mon Sep 17 00:00:00 2001 From: Payton McIntosh Date: Sun, 31 May 2026 19:57:52 +0100 Subject: [PATCH 02/32] Add act switch to generated test target Let generated projects run local workflow validation with `make test WITH_ACT=1` by mapping the Make flag to `RUN_ACT_VALIDATION=1` for the pytest invocation. Document the flag in generated `AGENTS.md` and assert the rendered Makefile and guidance contracts for both pure-Python and Python/Rust variants. --- template/AGENTS.md.jinja | 3 +++ template/Makefile.jinja | 4 +++- tests/test_template.py | 26 ++++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/template/AGENTS.md.jinja b/template/AGENTS.md.jinja index a30823e..16ef690 100644 --- a/template/AGENTS.md.jinja +++ b/template/AGENTS.md.jinja @@ -69,6 +69,9 @@ - Formatting is correct and validated. - **For Python files:** - **Testing:** Passes all relevant unit and behavioural tests (`make test`). + Use `make test WITH_ACT=1` when local GitHub Actions workflow validation + should run through `act`; this sets `RUN_ACT_VALIDATION=1` for the pytest + invocation. - **Linting:** Passes lint checks (`make lint`). - **Formatting:** Adheres to formatting standards (`make check-fmt`; use `make fmt` to apply fixes). diff --git a/template/Makefile.jinja b/template/Makefile.jinja index 576e78d..9bc76cc 100644 --- a/template/Makefile.jinja +++ b/template/Makefile.jinja @@ -9,6 +9,8 @@ USER_BIN_PATH := $(HOME)/.cargo/bin:$(HOME)/.local/bin:$(HOME)/.bun/bin TOOLS = $(MDFORMAT_ALL) $(MDLINT) VENV_TOOLS = pytest UV_ENV = PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1 UV_CACHE_DIR=.uv-cache UV_TOOL_DIR=.uv-tools +WITH_ACT ?= 0 +ACT_TEST_ENV = $(if $(filter 1 true yes on,$(WITH_ACT)),RUN_ACT_VALIDATION=1,) PYTEST_XDIST_WORKERS ?= auto PYTHON_TARGETS ?= {{ package_name }} tests PYLINT_PYTHON ?= pypy @@ -156,7 +158,7 @@ nixie: ## Validate Mermaid diagrams $(NIXIE) --no-sandbox test: build $(VENV_TOOLS) ## Run tests - $(UV_ENV) $(UV) run pytest -v -n $(PYTEST_XDIST_WORKERS) + $(UV_ENV) $(ACT_TEST_ENV) $(UV) run pytest -v -n $(PYTEST_XDIST_WORKERS) {% if use_rust %} @test -n "$(CARGO_AVAILABLE)" || { \ printf "Error: cargo is required for Rust tests, but '%s' was not found on PATH\n" "$(CARGO)" >&2; \ diff --git a/tests/test_template.py b/tests/test_template.py index e12b38a..6bf00d1 100644 --- a/tests/test_template.py +++ b/tests/test_template.py @@ -498,6 +498,7 @@ def test_generated_tooling_contracts( project.run("uv tool run mbake validate Makefile") pyproject = parse_toml_file(project / "pyproject.toml") + agents = read_generated_text(project / "AGENTS.md") makefile = read_generated_text(project / "Makefile") ci_workflow = read_generated_text(project / ".github" / "workflows" / "ci.yml") release_workflow = read_generated_text( @@ -515,6 +516,7 @@ def test_generated_tooling_contracts( assert_generated_tooling_contracts( package_name=package_name, + agents=agents, pyproject=pyproject, makefile=makefile, ci_workflow=ci_workflow, @@ -593,6 +595,7 @@ def test_generated_github_workflows_match_act_validation_contract( def assert_generated_tooling_contracts( *, package_name: str, + agents: str, pyproject: dict[str, Any], makefile: str, ci_workflow: str, @@ -608,6 +611,7 @@ def assert_generated_tooling_contracts( pyproject=pyproject, use_rust=use_rust, ) + assert_agents_contracts(agents) assert_makefile_contracts(makefile=makefile, use_rust=use_rust) assert_ci_workflow_contracts(ci_workflow=ci_workflow, use_rust=use_rust) assert_release_workflow_contracts( @@ -668,9 +672,31 @@ def assert_pyproject_contracts( ) +def assert_agents_contracts(agents: str) -> None: + """Assert generated assistant guidance documents act-enabled testing.""" + assert "make test WITH_ACT=1" in agents, ( + "expected generated AGENTS.md to document act-enabled test runs" + ) + assert "RUN_ACT_VALIDATION=1" in agents, ( + "expected generated AGENTS.md to describe the pytest act environment" + ) + + def assert_makefile_contracts(*, makefile: str, use_rust: bool) -> None: """Assert generated Makefile contracts for both template variants.""" assert_common_make_targets(makefile) + assert "WITH_ACT ?= 0" in makefile, ( + "expected generated Makefile to default act validation off" + ) + assert "ACT_TEST_ENV =" in makefile, ( + "expected generated Makefile to map WITH_ACT to pytest environment" + ) + assert "RUN_ACT_VALIDATION=1" in makefile, ( + "expected generated Makefile to enable act validation for pytest" + ) + assert "$(UV_ENV) $(ACT_TEST_ENV) $(UV) run pytest" in makefile, ( + "expected generated test target to include the act test environment" + ) assert "PYTHON_TARGETS ?=" in makefile, ( "expected generated Makefile to define Python target selection" ) From 958ee77b913951ae1ea18540112566e64dcbb4b8 Mon Sep 17 00:00:00 2001 From: Payton McIntosh Date: Sun, 31 May 2026 20:05:36 +0100 Subject: [PATCH 03/32] Add parent template test Makefile Expose `make test` in the parent repository so template tests run through `uvx` with the required pytest-copier, PyYAML, and syrupy dependencies. Document the Makefile entrypoint in the README and suppress nested Make directory messages so rendered-project snapshots stay stable when tests run from the parent target. --- Makefile | 10 ++++++++++ README.md | 15 +++++++++------ 2 files changed, 19 insertions(+), 6 deletions(-) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..ccde514 --- /dev/null +++ b/Makefile @@ -0,0 +1,10 @@ +.PHONY: help test + +MAKEFLAGS += --no-print-directory + +test: ## Run template tests + uvx --with pytest-copier --with pyyaml --with syrupy pytest tests/ + +help: ## Show available targets + @grep -E '^[a-zA-Z_-]+:.*?##' $(MAKEFILE_LIST) | \ + awk 'BEGIN {FS=":.*?## "; printf "Available targets:\n"} {printf " %-15s %s\n", $$1, $$2}' diff --git a/README.md b/README.md index 67eed41..149c882 100644 --- a/README.md +++ b/README.md @@ -14,16 +14,19 @@ The test suite relies on the `pytest-copier` plugin and renders generated projects that run Ruff, Pylint via a PyPy-backed runner, `ty`, pytest, and, when the Rust extension is enabled, Clippy, Whitaker, and nextest-aware Rust tests. -Install the `pytest-copier` test dependency before running this repository's -`pytest` suite. Generated projects install and run their own tooling, including -Ruff, Pylint via PyPy, `ty`, pytest, and, when Rust is enabled, Clippy, Whitaker, -and nextest. +Run the parent template tests through the repository `Makefile`. The `test` +target uses `uvx` to provide `pytest-copier`, `PyYAML`, and `syrupy` without a +manually managed virtual environment: ```bash -pip install pytest-copier +make test ``` -You can also run `scripts/setup_test_deps.sh` to install them automatically. +Generated projects install and run their own tooling, including Ruff, Pylint via +PyPy, `ty`, pytest, and, when Rust is enabled, Clippy, Whitaker, and nextest. + +You can also run `scripts/setup_test_deps.sh` to install parent test +dependencies into the current Python environment manually. ## Generated Quality Gate Flow From 1e8f36179511127ea5606fb78e224a19ee1d122f Mon Sep 17 00:00:00 2001 From: Payton McIntosh Date: Sun, 31 May 2026 22:21:37 +0100 Subject: [PATCH 04/32] Address workflow validation review comments Make release asset uploads idempotent by passing `--clobber` to `gh release upload` in the generated release workflow. Expand the test package documentation and move generated-file parsing and workflow contract assertions into dedicated helper modules so `tests/test_template.py` remains focused on top-level test cases. --- template/.github/workflows/release.yml.jinja | 2 +- tests/__init__.py | 26 +- tests/helpers/__init__.py | 1 + tests/helpers/generated_files.py | 175 +++++++ tests/helpers/rendering.py | 47 ++ tests/helpers/tooling_contracts.py | 292 +++++++++++ tests/test_template.py | 503 +------------------ 7 files changed, 560 insertions(+), 486 deletions(-) create mode 100644 tests/helpers/__init__.py create mode 100644 tests/helpers/generated_files.py create mode 100644 tests/helpers/rendering.py create mode 100644 tests/helpers/tooling_contracts.py diff --git a/template/.github/workflows/release.yml.jinja b/template/.github/workflows/release.yml.jinja index 4b47466..5d10ea1 100644 --- a/template/.github/workflows/release.yml.jinja +++ b/template/.github/workflows/release.yml.jinja @@ -62,6 +62,6 @@ jobs: run: | set -eu find dist/wheels-* -type f -name "*.whl" -print0 | \ - xargs -0 -r gh release upload "${{ "{{" }} github.ref_name {{ "}}" }}" + xargs -0 -r gh release upload "${{ "{{" }} github.ref_name {{ "}}" }}" --clobber env: GITHUB_TOKEN: ${{ "{{" }} secrets.GITHUB_TOKEN {{ "}}" }} diff --git a/tests/__init__.py b/tests/__init__.py index 7f1a999..3b56526 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1 +1,25 @@ -"""Test support package for the Copier template validation suite.""" +"""Support utilities for the Copier template validation test suite. + +The package exposes shared helpers and fixtures used by the repository tests, +including rendered-file parsers, generated-project command helpers, workflow +contract assertions, container runtime environment helpers, and pytest fixtures +loaded from :mod:`tests.conftest`. + +Import the package directly when a test only needs package discovery, or import +specific utilities from their modules when using them in assertions. Importing +``tests`` has no side effects beyond normal package initialisation; runtime +probing for tools such as Docker, Podman, or ``act`` stays inside fixtures and +helper functions. + +Examples +-------- +Use helpers from the package in template tests:: + + import tests + from tests.helpers.generated_files import parse_yaml_mapping + from tests.utilities import docker_environment + + assert tests.__doc__ + workflow = parse_yaml_mapping("name: CI\n", "workflow") + env = docker_environment() +""" diff --git a/tests/helpers/__init__.py b/tests/helpers/__init__.py new file mode 100644 index 0000000..fa520bd --- /dev/null +++ b/tests/helpers/__init__.py @@ -0,0 +1 @@ +"""Shared helper modules for rendered Copier template tests.""" diff --git a/tests/helpers/generated_files.py b/tests/helpers/generated_files.py new file mode 100644 index 0000000..744644a --- /dev/null +++ b/tests/helpers/generated_files.py @@ -0,0 +1,175 @@ +"""Parse and validate generated project files in template tests.""" + +from __future__ import annotations + +import tomllib +from pathlib import Path +from typing import Any + +import pytest +import yaml + + +def read_generated_text(path: Path) -> str: + """Read a generated file with assertion-focused error context. + + Parameters + ---------- + path + Path to the generated file to read. + + Returns + ------- + str + UTF-8 decoded file contents. + + Raises + ------ + pytest.fail.Exception + Raised when the file cannot be read. + + Examples + -------- + Read a rendered workflow before parsing it:: + + workflow = read_generated_text(project / ".github/workflows/ci.yml") + """ + try: + return path.read_text(encoding="utf-8") + except OSError as error: + pytest.fail(f"could not read generated file {path}: {error}") + + +def parse_toml_file(path: Path) -> dict[str, Any]: + """Parse generated TOML with assertion-focused error context. + + Parameters + ---------- + path + Path to the generated TOML file. + + Returns + ------- + dict[str, Any] + Parsed TOML mapping. + + Raises + ------ + pytest.fail.Exception + Raised when the file cannot be read or the TOML is invalid. + + Examples + -------- + Parse generated project metadata:: + + pyproject = parse_toml_file(project / "pyproject.toml") + """ + text = read_generated_text(path) + try: + parsed = tomllib.loads(text) + except tomllib.TOMLDecodeError as error: + pytest.fail(f"could not parse generated TOML {path}: {error}") + return parsed + + +def parse_yaml_mapping(text: str, label: str) -> dict[str, Any]: + """Parse generated YAML as a mapping with clear failure context. + + Parameters + ---------- + text + YAML document text to parse. + label + Human-readable label used in assertion failure messages. + + Returns + ------- + dict[str, Any] + Parsed YAML mapping. + + Raises + ------ + pytest.fail.Exception + Raised when the YAML is invalid or the parsed document is not a mapping. + + Examples + -------- + Parse a rendered CI workflow:: + + workflow = parse_yaml_mapping(ci_workflow, "CI workflow") + """ + try: + parsed = yaml.safe_load(text) + except yaml.YAMLError as error: + pytest.fail(f"could not parse generated {label}: {error}") + if not isinstance(parsed, dict): + pytest.fail(f"expected generated {label} to parse as a mapping") + return parsed + + +def require_mapping(mapping: dict[str, Any], key: str, label: str) -> dict[str, Any]: + """Return a nested mapping or fail with the missing schema path. + + Parameters + ---------- + mapping + Parent mapping to inspect. + key + Nested key expected to contain a mapping. + label + Human-readable schema path used in assertion failure messages. + + Returns + ------- + dict[str, Any] + Nested mapping value. + + Raises + ------ + pytest.fail.Exception + Raised when the key is missing or the value is not a mapping. + + Examples + -------- + Extract workflow jobs after parsing YAML:: + + jobs = require_mapping(workflow, "jobs", "CI workflow") + """ + value = mapping.get(key) + if not isinstance(value, dict): + pytest.fail(f"expected {label} to include mapping key {key!r}") + return value + + +def require_sequence(mapping: dict[str, Any], key: str, label: str) -> list[Any]: + """Return a nested sequence or fail with the missing schema path. + + Parameters + ---------- + mapping + Parent mapping to inspect. + key + Nested key expected to contain a sequence. + label + Human-readable schema path used in assertion failure messages. + + Returns + ------- + list[Any] + Nested sequence value. + + Raises + ------ + pytest.fail.Exception + Raised when the key is missing or the value is not a sequence. + + Examples + -------- + Extract workflow steps from a parsed job:: + + steps = require_sequence(job, "steps", "CI lint-test job") + """ + value = mapping.get(key) + if not isinstance(value, list): + pytest.fail(f"expected {label} to include sequence key {key!r}") + return value diff --git a/tests/helpers/rendering.py b/tests/helpers/rendering.py new file mode 100644 index 0000000..6c7e4e7 --- /dev/null +++ b/tests/helpers/rendering.py @@ -0,0 +1,47 @@ +"""Render generated projects and run their public commands in tests.""" + +from __future__ import annotations + +import shlex +from pathlib import Path + +from pytest_copier.plugin import CopierFixture, CopierProject + + +def run_quality_gates(project: CopierProject) -> None: + """Run the rendered project's public quality gate.""" + project.run("make all") + + +def render_project( + tmp_path: Path, + copier: CopierFixture, + *, + project_name: str, + package_name: str, + use_rust: bool = False, + python_version: str = "3.10", +) -> CopierProject: + """Render a generated Python project with explicit template answers.""" + return copier.copy( + tmp_path, + project_name=project_name, + package_name=package_name, + use_rust=use_rust, + python_version=python_version, + ) + + +def check_generated_import(project: CopierProject, package: str, greeting: str) -> None: + """Import a generated package and assert its greeting.""" + script = ( + "import importlib; " + f"module = importlib.import_module({package!r}); " + f"assert module.hello() == {greeting!r}" + ) + project.run(f"uv run python -c {shlex.quote(script)}") + + +def read_generated_file(project: CopierProject, relative_path: str) -> str: + """Read a rendered project file as UTF-8 text.""" + return (project / relative_path).read_text(encoding="utf-8") diff --git a/tests/helpers/tooling_contracts.py b/tests/helpers/tooling_contracts.py new file mode 100644 index 0000000..db506a4 --- /dev/null +++ b/tests/helpers/tooling_contracts.py @@ -0,0 +1,292 @@ +"""Assertion helpers for generated tooling and workflow contracts.""" + +from __future__ import annotations + +from typing import Any + +from tests.helpers.generated_files import ( + parse_yaml_mapping, + require_mapping, + require_sequence, +) + + +def assert_common_make_targets(makefile: str) -> None: + """Assert Makefile targets shared by all generated variants.""" + assert "lint-python: build" in makefile, "Makefile should expose lint-python" + assert "lint: lint-python" in makefile, "lint should delegate to lint-python" + assert ".uv-cache .uv-tools" in makefile, "clean should remove uv state dirs" + + +def assert_generated_tooling_contracts( + *, + package_name: str, + agents: str, + pyproject: dict[str, Any], + makefile: str, + ci_workflow: str, + release_workflow: str, + build_wheels_workflow: str, + build_wheels_action: str, + pure_wheel_action: str, + use_rust: bool, +) -> None: + """Assert generated Python/Rust tooling contracts from one validator.""" + assert_pyproject_contracts( + package_name=package_name, + pyproject=pyproject, + use_rust=use_rust, + ) + assert_agents_contracts(agents) + assert_makefile_contracts(makefile=makefile, use_rust=use_rust) + assert_ci_workflow_contracts(ci_workflow=ci_workflow, use_rust=use_rust) + assert_release_workflow_contracts( + release_workflow=release_workflow, + use_rust=use_rust, + ) + assert_wheel_workflow_contracts( + build_wheels_workflow=build_wheels_workflow, + build_wheels_action=build_wheels_action, + pure_wheel_action=pure_wheel_action, + ) + + +def assert_pyproject_contracts( + *, package_name: str, pyproject: dict[str, Any], use_rust: bool +) -> None: + """Assert generated Python packaging contracts.""" + project = require_mapping(pyproject, "project", "pyproject.toml") + assert project.get("name"), "expected generated project metadata to include a name" + assert project.get("requires-python") == ">=3.10", ( + "expected generated pyproject.toml to use the requested Python version" + ) + dependency_groups = require_mapping( + pyproject, + "dependency-groups", + "pyproject.toml", + ) + dev_dependencies = dependency_groups.get("dev") + assert isinstance(dev_dependencies, list), ( + "expected generated pyproject.toml to include a dev dependency group" + ) + for dependency in ["pytest", "ruff", "pyright", "ty", "pytest-xdist"]: + assert dependency in dev_dependencies, ( + f"expected generated dev dependencies to include {dependency}" + ) + + build_system = require_mapping(pyproject, "build-system", "pyproject.toml") + if use_rust: + assert build_system.get("build-backend") == "maturin", ( + "expected Rust variant to use maturin as the build backend" + ) + maturin = require_mapping(pyproject, "tool", "pyproject.toml").get("maturin") + assert isinstance(maturin, dict), ( + "expected Rust variant to include tool.maturin configuration" + ) + assert maturin.get("manifest-path") == "rust_extension/Cargo.toml", ( + "expected Rust variant to point maturin at the extension manifest" + ) + assert maturin.get("module-name") == f"{package_name}._{package_name}_rs", ( + "expected Rust variant to render the package-specific module name" + ) + else: + assert build_system.get("build-backend") == "hatchling.build", ( + "expected pure-Python variant to use hatchling as the build backend" + ) + assert "maturin" not in pyproject.get("tool", {}), ( + "expected pure-Python variant to omit maturin configuration" + ) + + +def assert_agents_contracts(agents: str) -> None: + """Assert generated assistant guidance documents act-enabled testing.""" + assert "make test WITH_ACT=1" in agents, ( + "expected generated AGENTS.md to document act-enabled test runs" + ) + assert "RUN_ACT_VALIDATION=1" in agents, ( + "expected generated AGENTS.md to describe the pytest act environment" + ) + + +def assert_makefile_contracts(*, makefile: str, use_rust: bool) -> None: + """Assert generated Makefile contracts for both template variants.""" + assert_common_make_targets(makefile) + assert "WITH_ACT ?= 0" in makefile, ( + "expected generated Makefile to default act validation off" + ) + assert "ACT_TEST_ENV =" in makefile, ( + "expected generated Makefile to map WITH_ACT to pytest environment" + ) + assert "RUN_ACT_VALIDATION=1" in makefile, ( + "expected generated Makefile to enable act validation for pytest" + ) + assert "$(UV_ENV) $(ACT_TEST_ENV) $(UV) run pytest" in makefile, ( + "expected generated test target to include the act test environment" + ) + assert "PYTHON_TARGETS ?=" in makefile, ( + "expected generated Makefile to define Python target selection" + ) + assert "PYLINT_PYPY_SHIM_REF ?=" in makefile, ( + "expected generated Makefile to expose the Pylint shim revision" + ) + assert "test: build $(VENV_TOOLS)" in makefile, ( + "expected generated Makefile test target to depend on the project env" + ) + if use_rust: + assert "TEST_CMD :=" in makefile, ( + "expected Rust variant to select nextest or cargo test" + ) + assert "lint-rust: build whitaker" in makefile, ( + "expected Rust variant to expose the Rust lint target" + ) + assert "cargo is required for Rust tests" in makefile, ( + "expected Rust variant to fail clearly without cargo" + ) + assert "$(CARGO) $(TEST_CMD) $(TEST_FLAGS)" in makefile, ( + "expected Rust variant tests to use the selected cargo test command" + ) + else: + assert "lint-rust" not in makefile, ( + "expected pure-Python variant to omit Rust lint targets" + ) + assert "TEST_CMD :=" not in makefile, ( + "expected pure-Python variant to omit Rust test command selection" + ) + + +def assert_ci_workflow_contracts(*, ci_workflow: str, use_rust: bool) -> None: + """Assert generated CI workflow contracts.""" + parsed_ci_workflow = parse_yaml_mapping(ci_workflow, "CI workflow") + jobs = require_mapping(parsed_ci_workflow, "jobs", "CI workflow") + lint_test = require_mapping(jobs, "lint-test", "CI workflow jobs") + steps = require_sequence(lint_test, "steps", "CI lint-test job") + assert checkout_steps_disable_credentials(steps), ( + "expected generated CI workflow to disable checkout credentials" + ) + assert "make check-fmt" in ci_workflow, ( + "expected generated CI workflow to run the formatting gate" + ) + assert "make lint" in ci_workflow, ( + "expected generated CI workflow to run the lint gate" + ) + assert "make typecheck" in ci_workflow, ( + "expected generated CI workflow to run the typecheck gate" + ) + assert "make build" in ci_workflow, ( + "expected generated CI workflow to build the project before checks" + ) + assert "leynos/shared-actions/.github/actions/generate-coverage" in ci_workflow, ( + "expected generated CI workflow to use the shared coverage action" + ) + if use_rust: + assert "leynos/shared-actions/.github/actions/setup-rust" in ci_workflow, ( + "expected Rust variant CI to set up Rust" + ) + assert "Cache Rust lint and test tools" in ci_workflow, ( + "expected Rust variant CI to cache Rust tools" + ) + assert "cargo-manifest: rust_extension/Cargo.toml" in ci_workflow, ( + "expected Rust variant CI to pass the Rust manifest to coverage" + ) + else: + assert "setup-rust" not in ci_workflow, ( + "expected pure-Python CI to omit Rust setup" + ) + assert "cargo-manifest" not in ci_workflow, ( + "expected pure-Python CI to omit Rust coverage inputs" + ) + + +def assert_release_workflow_contracts(*, release_workflow: str, use_rust: bool) -> None: + """Assert generated release workflow contracts.""" + parsed_release_workflow = parse_yaml_mapping(release_workflow, "release workflow") + jobs = require_mapping(parsed_release_workflow, "jobs", "release workflow") + release = require_mapping(jobs, "release", "release workflow jobs") + release_steps = require_sequence(release, "steps", "release job") + assert checkout_steps_disable_credentials(release_steps), ( + "expected generated release workflow to disable checkout credentials" + ) + assert "softprops/action-gh-release@v2" in release_workflow, ( + "expected release workflow to create a GitHub release" + ) + assert "actions/download-artifact@v4" in release_workflow, ( + "expected release workflow to download wheel artefacts" + ) + assert "--clobber" in release_workflow, ( + "expected release workflow to overwrite existing wheel assets on rerun" + ) + if use_rust: + assert "build-wheels" in jobs, ( + "expected Rust variant release workflow to build platform wheels" + ) + build_wheels = require_mapping(jobs, "build-wheels", "release workflow jobs") + assert build_wheels.get("uses") == "./.github/workflows/build-wheels.yml", ( + "expected Rust variant release workflow to call build-wheels.yml" + ) + assert release.get("needs") == ["build-wheels"], ( + "expected Rust variant release job to wait for platform wheels" + ) + assert "pure-wheel" not in jobs, ( + "expected Rust variant release workflow to skip pure wheel job" + ) + else: + assert "pure-wheel" in jobs, ( + "expected pure-Python release workflow to build one pure wheel" + ) + assert release.get("needs") == ["pure-wheel"], ( + "expected pure-Python release job to wait for the pure wheel" + ) + assert "build-wheels" not in jobs, ( + "expected pure-Python release workflow to skip platform wheel matrix" + ) + + +def assert_wheel_workflow_contracts( + *, + build_wheels_workflow: str, + build_wheels_action: str, + pure_wheel_action: str, +) -> None: + """Assert generated wheel workflow and action contracts.""" + parsed_build_wheels = parse_yaml_mapping( + build_wheels_workflow, + "build-wheels workflow", + ) + build_jobs = require_mapping(parsed_build_wheels, "jobs", "build-wheels workflow") + build_job = require_mapping(build_jobs, "build", "build-wheels workflow jobs") + strategy = require_mapping(build_job, "strategy", "build-wheels build job") + matrix = require_mapping(strategy, "matrix", "build-wheels strategy") + includes = require_sequence(matrix, "include", "build-wheels matrix") + assert len(includes) == 6, ( + "expected build-wheels workflow to cover Linux, Windows, and macOS" + ) + assert "persist-credentials: false" in build_wheels_workflow, ( + "expected build-wheels workflow checkout to disable credentials" + ) + assert "uvx --with 'cibuildwheel>=2.16.0,<4.0.0' cibuildwheel" in ( + build_wheels_action + ), "expected Rust wheel action to build through cibuildwheel" + assert "persist-credentials: false" in build_wheels_action, ( + "expected build-wheels action checkout to disable credentials" + ) + assert "uv build --wheel" in pure_wheel_action, ( + "expected pure-wheel action to build through uv" + ) + assert "persist-credentials: false" in pure_wheel_action, ( + "expected pure-wheel action checkout to disable credentials" + ) + + +def checkout_steps_disable_credentials(steps: list[Any]) -> bool: + """Return whether all checkout steps disable credential persistence.""" + checkout_steps = [ + step + for step in steps + if isinstance(step, dict) + and str(step.get("uses", "")).startswith("actions/checkout@") + ] + return bool(checkout_steps) and all( + isinstance(step.get("with"), dict) + and step["with"].get("persist-credentials") is False + for step in checkout_steps + ) diff --git a/tests/test_template.py b/tests/test_template.py index 6bf00d1..6832322 100644 --- a/tests/test_template.py +++ b/tests/test_template.py @@ -22,224 +22,30 @@ from __future__ import annotations -import shlex -import tomllib from pathlib import Path -from typing import Any import pytest -import yaml -from pytest_copier.plugin import CopierFixture, CopierProject +from pytest_copier.plugin import CopierFixture from syrupy.assertion import SnapshotAssertion - -def run_quality_gates(project: CopierProject) -> None: - """Run the rendered project's public quality gate. - - Parameters - ---------- - project - Rendered ``pytest-copier`` project whose root contains the generated - Makefile. - - Returns - ------- - None - The helper delegates validation to the generated project's ``make all`` - target. - - Raises - ------ - AssertionError - Raised by ``pytest-copier`` if the command exits unsuccessfully. - - Examples - -------- - Validate a rendered project before making assertions about its files:: - - project = copier.copy(tmp_path / "pure", use_rust=False) - run_quality_gates(project) - """ - project.run("make all") - - -def render_project( - tmp_path: Path, - copier: CopierFixture, - *, - project_name: str, - package_name: str, - use_rust: bool = False, - python_version: str = "3.10", -) -> CopierProject: - """Render a generated Python project with explicit template answers. - - Parameters - ---------- - tmp_path - Temporary directory used as the generated project destination. - copier - ``pytest-copier`` fixture bound to this template repository. - project_name - Project name answer passed to Copier. - package_name - Package name answer passed to Copier. - use_rust - Whether to include the optional PyO3 extension. - python_version - Minimum supported Python version answer passed to Copier. - - Returns - ------- - CopierProject - Rendered project wrapper for file assertions and command execution. - """ - return copier.copy( - tmp_path, - project_name=project_name, - package_name=package_name, - use_rust=use_rust, - python_version=python_version, - ) - - -def check_generated_import(project: CopierProject, package: str, greeting: str) -> None: - """Import a generated package and assert its greeting. - - Parameters - ---------- - project - Rendered project whose managed environment should contain the package. - package - Import name to load through ``importlib.import_module``. - greeting - Expected return value from the generated package's ``hello`` function. - - Returns - ------- - None - The helper succeeds when the generated import and assertion succeed. - - Raises - ------ - AssertionError - Raised by ``pytest-copier`` if the command exits unsuccessfully. - - Examples - -------- - Check the generated pure-Python package import path:: - - check_generated_import(project, "pure_pkg", "hello from Python") - """ - script = ( - "import importlib; " - f"module = importlib.import_module({package!r}); " - f"assert module.hello() == {greeting!r}" - ) - project.run(f"uv run python -c {shlex.quote(script)}") - - -def read_generated_file(project: CopierProject, relative_path: str) -> str: - """Read a rendered project file as UTF-8 text. - - Parameters - ---------- - project - Rendered ``pytest-copier`` project to read from. - relative_path - Path to the generated file, relative to the rendered project root. - - Returns - ------- - str - File contents decoded as UTF-8. - - Raises - ------ - FileNotFoundError - Raised if the requested generated file does not exist. - - Examples - -------- - Read the generated Makefile for target assertions:: - - makefile = read_generated_file(project, "Makefile") - """ - return (project / relative_path).read_text(encoding="utf-8") - - -def read_generated_text(path: Path) -> str: - """Read a generated file with assertion-focused error context.""" - try: - return path.read_text(encoding="utf-8") - except OSError as error: - pytest.fail(f"could not read generated file {path}: {error}") - - -def parse_toml_file(path: Path) -> dict[str, Any]: - """Parse generated TOML with assertion-focused error context.""" - text = read_generated_text(path) - try: - parsed = tomllib.loads(text) - except tomllib.TOMLDecodeError as error: - pytest.fail(f"could not parse generated TOML {path}: {error}") - return parsed - - -def parse_yaml_mapping(text: str, label: str) -> dict[str, Any]: - """Parse generated YAML as a mapping with clear failure context.""" - try: - parsed = yaml.safe_load(text) - except yaml.YAMLError as error: - pytest.fail(f"could not parse generated {label}: {error}") - if not isinstance(parsed, dict): - pytest.fail(f"expected generated {label} to parse as a mapping") - return parsed - - -def require_mapping(mapping: dict[str, Any], key: str, label: str) -> dict[str, Any]: - """Return a nested mapping or fail with the missing schema path.""" - value = mapping.get(key) - if not isinstance(value, dict): - pytest.fail(f"expected {label} to include mapping key {key!r}") - return value - - -def require_sequence(mapping: dict[str, Any], key: str, label: str) -> list[Any]: - """Return a nested sequence or fail with the missing schema path.""" - value = mapping.get(key) - if not isinstance(value, list): - pytest.fail(f"expected {label} to include sequence key {key!r}") - return value - - -def assert_common_make_targets(makefile: str) -> None: - """Assert Makefile targets shared by all generated variants. - - Parameters - ---------- - makefile - UTF-8 text of a generated Makefile. - - Returns - ------- - None - The helper returns after all common target assertions pass. - - Raises - ------ - AssertionError - Raised when a required shared target or cleanup path is missing. - - Examples - -------- - Validate shared targets after reading a generated Makefile:: - - assert_common_make_targets(makefile) - """ - assert "lint-python: build" in makefile, "Makefile should expose lint-python" - assert "lint: lint-python" in makefile, "lint should delegate to lint-python" - assert ".uv-cache .uv-tools" in makefile, "clean should remove uv state dirs" +from tests.helpers.generated_files import ( + parse_toml_file, + parse_yaml_mapping, + read_generated_text, + require_mapping, + require_sequence, +) +from tests.helpers.rendering import ( + check_generated_import, + read_generated_file, + render_project, + run_quality_gates, +) +from tests.helpers.tooling_contracts import ( + assert_common_make_targets, + assert_generated_tooling_contracts, + checkout_steps_disable_credentials, +) def test_python_only_help_output_snapshot( @@ -590,274 +396,3 @@ def test_generated_github_workflows_match_act_validation_contract( assert "cargo-manifest" not in coverage_inputs, ( "expected pure-Python variant to omit Rust coverage inputs" ) - - -def assert_generated_tooling_contracts( - *, - package_name: str, - agents: str, - pyproject: dict[str, Any], - makefile: str, - ci_workflow: str, - release_workflow: str, - build_wheels_workflow: str, - build_wheels_action: str, - pure_wheel_action: str, - use_rust: bool, -) -> None: - """Assert generated Python/Rust tooling contracts from one validator.""" - assert_pyproject_contracts( - package_name=package_name, - pyproject=pyproject, - use_rust=use_rust, - ) - assert_agents_contracts(agents) - assert_makefile_contracts(makefile=makefile, use_rust=use_rust) - assert_ci_workflow_contracts(ci_workflow=ci_workflow, use_rust=use_rust) - assert_release_workflow_contracts( - release_workflow=release_workflow, - use_rust=use_rust, - ) - assert_wheel_workflow_contracts( - build_wheels_workflow=build_wheels_workflow, - build_wheels_action=build_wheels_action, - pure_wheel_action=pure_wheel_action, - ) - - -def assert_pyproject_contracts( - *, package_name: str, pyproject: dict[str, Any], use_rust: bool -) -> None: - """Assert generated Python packaging contracts.""" - project = require_mapping(pyproject, "project", "pyproject.toml") - assert project.get("name"), "expected generated project metadata to include a name" - assert project.get("requires-python") == ">=3.10", ( - "expected generated pyproject.toml to use the requested Python version" - ) - dependency_groups = require_mapping( - pyproject, - "dependency-groups", - "pyproject.toml", - ) - dev_dependencies = dependency_groups.get("dev") - assert isinstance(dev_dependencies, list), ( - "expected generated pyproject.toml to include a dev dependency group" - ) - for dependency in ["pytest", "ruff", "pyright", "ty", "pytest-xdist"]: - assert dependency in dev_dependencies, ( - f"expected generated dev dependencies to include {dependency}" - ) - - build_system = require_mapping(pyproject, "build-system", "pyproject.toml") - if use_rust: - assert build_system.get("build-backend") == "maturin", ( - "expected Rust variant to use maturin as the build backend" - ) - maturin = require_mapping(pyproject, "tool", "pyproject.toml").get("maturin") - assert isinstance(maturin, dict), ( - "expected Rust variant to include tool.maturin configuration" - ) - assert maturin.get("manifest-path") == "rust_extension/Cargo.toml", ( - "expected Rust variant to point maturin at the extension manifest" - ) - assert maturin.get("module-name") == f"{package_name}._{package_name}_rs", ( - "expected Rust variant to render the package-specific module name" - ) - else: - assert build_system.get("build-backend") == "hatchling.build", ( - "expected pure-Python variant to use hatchling as the build backend" - ) - assert "maturin" not in pyproject.get("tool", {}), ( - "expected pure-Python variant to omit maturin configuration" - ) - - -def assert_agents_contracts(agents: str) -> None: - """Assert generated assistant guidance documents act-enabled testing.""" - assert "make test WITH_ACT=1" in agents, ( - "expected generated AGENTS.md to document act-enabled test runs" - ) - assert "RUN_ACT_VALIDATION=1" in agents, ( - "expected generated AGENTS.md to describe the pytest act environment" - ) - - -def assert_makefile_contracts(*, makefile: str, use_rust: bool) -> None: - """Assert generated Makefile contracts for both template variants.""" - assert_common_make_targets(makefile) - assert "WITH_ACT ?= 0" in makefile, ( - "expected generated Makefile to default act validation off" - ) - assert "ACT_TEST_ENV =" in makefile, ( - "expected generated Makefile to map WITH_ACT to pytest environment" - ) - assert "RUN_ACT_VALIDATION=1" in makefile, ( - "expected generated Makefile to enable act validation for pytest" - ) - assert "$(UV_ENV) $(ACT_TEST_ENV) $(UV) run pytest" in makefile, ( - "expected generated test target to include the act test environment" - ) - assert "PYTHON_TARGETS ?=" in makefile, ( - "expected generated Makefile to define Python target selection" - ) - assert "PYLINT_PYPY_SHIM_REF ?=" in makefile, ( - "expected generated Makefile to expose the Pylint shim revision" - ) - assert "test: build $(VENV_TOOLS)" in makefile, ( - "expected generated Makefile test target to depend on the project env" - ) - if use_rust: - assert "TEST_CMD :=" in makefile, ( - "expected Rust variant to select nextest or cargo test" - ) - assert "lint-rust: build whitaker" in makefile, ( - "expected Rust variant to expose the Rust lint target" - ) - assert "cargo is required for Rust tests" in makefile, ( - "expected Rust variant to fail clearly without cargo" - ) - assert "$(CARGO) $(TEST_CMD) $(TEST_FLAGS)" in makefile, ( - "expected Rust variant tests to use the selected cargo test command" - ) - else: - assert "lint-rust" not in makefile, ( - "expected pure-Python variant to omit Rust lint targets" - ) - assert "TEST_CMD :=" not in makefile, ( - "expected pure-Python variant to omit Rust test command selection" - ) - - -def assert_ci_workflow_contracts(*, ci_workflow: str, use_rust: bool) -> None: - """Assert generated CI workflow contracts.""" - parsed_ci_workflow = parse_yaml_mapping(ci_workflow, "CI workflow") - jobs = require_mapping(parsed_ci_workflow, "jobs", "CI workflow") - lint_test = require_mapping(jobs, "lint-test", "CI workflow jobs") - steps = require_sequence(lint_test, "steps", "CI lint-test job") - assert checkout_steps_disable_credentials(steps), ( - "expected generated CI workflow to disable checkout credentials" - ) - assert "make check-fmt" in ci_workflow, ( - "expected generated CI workflow to run the formatting gate" - ) - assert "make lint" in ci_workflow, ( - "expected generated CI workflow to run the lint gate" - ) - assert "make typecheck" in ci_workflow, ( - "expected generated CI workflow to run the typecheck gate" - ) - assert "make build" in ci_workflow, ( - "expected generated CI workflow to build the project before checks" - ) - assert "leynos/shared-actions/.github/actions/generate-coverage" in ci_workflow, ( - "expected generated CI workflow to use the shared coverage action" - ) - if use_rust: - assert "leynos/shared-actions/.github/actions/setup-rust" in ci_workflow, ( - "expected Rust variant CI to set up Rust" - ) - assert "Cache Rust lint and test tools" in ci_workflow, ( - "expected Rust variant CI to cache Rust tools" - ) - assert "cargo-manifest: rust_extension/Cargo.toml" in ci_workflow, ( - "expected Rust variant CI to pass the Rust manifest to coverage" - ) - else: - assert "setup-rust" not in ci_workflow, ( - "expected pure-Python CI to omit Rust setup" - ) - assert "cargo-manifest" not in ci_workflow, ( - "expected pure-Python CI to omit Rust coverage inputs" - ) - - -def assert_release_workflow_contracts(*, release_workflow: str, use_rust: bool) -> None: - """Assert generated release workflow contracts.""" - parsed_release_workflow = parse_yaml_mapping(release_workflow, "release workflow") - jobs = require_mapping(parsed_release_workflow, "jobs", "release workflow") - release = require_mapping(jobs, "release", "release workflow jobs") - release_steps = require_sequence(release, "steps", "release job") - assert checkout_steps_disable_credentials(release_steps), ( - "expected generated release workflow to disable checkout credentials" - ) - assert "softprops/action-gh-release@v2" in release_workflow, ( - "expected release workflow to create a GitHub release" - ) - assert "actions/download-artifact@v4" in release_workflow, ( - "expected release workflow to download wheel artefacts" - ) - if use_rust: - assert "build-wheels" in jobs, ( - "expected Rust variant release workflow to build platform wheels" - ) - build_wheels = require_mapping(jobs, "build-wheels", "release workflow jobs") - assert build_wheels.get("uses") == "./.github/workflows/build-wheels.yml", ( - "expected Rust variant release workflow to call build-wheels.yml" - ) - assert release.get("needs") == ["build-wheels"], ( - "expected Rust variant release job to wait for platform wheels" - ) - assert "pure-wheel" not in jobs, ( - "expected Rust variant release workflow to skip pure wheel job" - ) - else: - assert "pure-wheel" in jobs, ( - "expected pure-Python release workflow to build one pure wheel" - ) - assert release.get("needs") == ["pure-wheel"], ( - "expected pure-Python release job to wait for the pure wheel" - ) - assert "build-wheels" not in jobs, ( - "expected pure-Python release workflow to skip platform wheel matrix" - ) - - -def assert_wheel_workflow_contracts( - *, - build_wheels_workflow: str, - build_wheels_action: str, - pure_wheel_action: str, -) -> None: - """Assert generated wheel workflow and action contracts.""" - parsed_build_wheels = parse_yaml_mapping( - build_wheels_workflow, - "build-wheels workflow", - ) - build_jobs = require_mapping(parsed_build_wheels, "jobs", "build-wheels workflow") - build_job = require_mapping(build_jobs, "build", "build-wheels workflow jobs") - strategy = require_mapping(build_job, "strategy", "build-wheels build job") - matrix = require_mapping(strategy, "matrix", "build-wheels strategy") - includes = require_sequence(matrix, "include", "build-wheels matrix") - assert len(includes) == 6, ( - "expected build-wheels workflow to cover Linux, Windows, and macOS" - ) - assert "persist-credentials: false" in build_wheels_workflow, ( - "expected build-wheels workflow checkout to disable credentials" - ) - assert "uvx --with 'cibuildwheel>=2.16.0,<4.0.0' cibuildwheel" in ( - build_wheels_action - ), "expected Rust wheel action to build through cibuildwheel" - assert "persist-credentials: false" in build_wheels_action, ( - "expected build-wheels action checkout to disable credentials" - ) - assert "uv build --wheel" in pure_wheel_action, ( - "expected pure-wheel action to build through uv" - ) - assert "persist-credentials: false" in pure_wheel_action, ( - "expected pure-wheel action checkout to disable credentials" - ) - - -def checkout_steps_disable_credentials(steps: list[Any]) -> bool: - """Return whether all checkout steps disable credential persistence.""" - checkout_steps = [ - step - for step in steps - if isinstance(step, dict) - and str(step.get("uses", "")).startswith("actions/checkout@") - ] - return bool(checkout_steps) and all( - isinstance(step.get("with"), dict) - and step["with"].get("persist-credentials") is False - for step in checkout_steps - ) From 527e70f632928641cb2df3e4c5843999ab53453b Mon Sep 17 00:00:00 2001 From: Payton McIntosh Date: Mon, 1 Jun 2026 12:12:13 +0100 Subject: [PATCH 05/32] Document rendered test helpers Expand public test and helper docstrings to follow the project NumPy-style convention. Keep subsidiary workflow contract assertions private so the helper module exposes only the functions consumed by top-level tests. Move the CI coverage-action assertion behind a helper to keep `tests/test_template.py` under the 400-line guideline after the docstring updates. --- tests/helpers/__init__.py | 28 ++++- tests/helpers/rendering.py | 93 +++++++++++++- tests/helpers/tooling_contracts.py | 187 ++++++++++++++++++++++++++--- tests/test_template.py | 102 ++++++++-------- 4 files changed, 336 insertions(+), 74 deletions(-) diff --git a/tests/helpers/__init__.py b/tests/helpers/__init__.py index fa520bd..2332239 100644 --- a/tests/helpers/__init__.py +++ b/tests/helpers/__init__.py @@ -1 +1,27 @@ -"""Shared helper modules for rendered Copier template tests.""" +"""Shared test utilities for rendered Copier template tests. + +The helpers package groups reusable support code for tests that render Copier +projects and inspect their generated files. It contains modules for generated +file parsing, rendered-project command execution, and tooling or workflow +contract assertions. Fixtures remain in :mod:`tests.conftest`, while runtime +container helpers remain in :mod:`tests.utilities`. + +Import helper functions from the module that owns their behaviour. Test modules +may also import common helpers from ``tests.helpers`` when this package re-exports +them, but module-level imports keep ownership clearer for larger assertions. + +Examples +-------- +Use the generated-file and contract helpers in a template test:: + + from tests.helpers.generated_files import parse_yaml_mapping + from tests.helpers.tooling_contracts import assert_ci_coverage_action_contract + + workflow = parse_yaml_mapping(ci_workflow, "CI workflow") + assert "jobs" in workflow + assert_ci_coverage_action_contract( + ci_workflow=ci_workflow, + package_name="example_pkg", + use_rust=False, + ) +""" diff --git a/tests/helpers/rendering.py b/tests/helpers/rendering.py index 6c7e4e7..e0f81b2 100644 --- a/tests/helpers/rendering.py +++ b/tests/helpers/rendering.py @@ -9,7 +9,25 @@ def run_quality_gates(project: CopierProject) -> None: - """Run the rendered project's public quality gate.""" + """Run the rendered project's public quality gate. + + Parameters + ---------- + project : CopierProject + Rendered ``pytest-copier`` project whose root contains the generated + Makefile. + + Returns + ------- + None + The helper returns after the generated ``make all`` target succeeds. + + Raises + ------ + AssertionError + Raised by ``pytest-copier`` when the generated command exits + unsuccessfully. + """ project.run("make all") @@ -22,7 +40,33 @@ def render_project( use_rust: bool = False, python_version: str = "3.10", ) -> CopierProject: - """Render a generated Python project with explicit template answers.""" + """Render a generated Python project with explicit template answers. + + Parameters + ---------- + tmp_path : pathlib.Path + Temporary directory used as the generated project destination. + copier : CopierFixture + ``pytest-copier`` fixture bound to this template repository. + project_name : str + Project name answer passed to Copier. + package_name : str + Python import package name answer passed to Copier. + use_rust : bool, default=False + Whether to include the optional PyO3 extension. + python_version : str, default="3.10" + Minimum supported Python version answer passed to Copier. + + Returns + ------- + CopierProject + Rendered project wrapper for file assertions and command execution. + + Raises + ------ + Exception + Propagates rendering failures raised by Copier or ``pytest-copier``. + """ return copier.copy( tmp_path, project_name=project_name, @@ -33,7 +77,28 @@ def render_project( def check_generated_import(project: CopierProject, package: str, greeting: str) -> None: - """Import a generated package and assert its greeting.""" + """Import a generated package and assert its greeting. + + Parameters + ---------- + project : CopierProject + Rendered project whose managed environment should contain the package. + package : str + Import name to load through ``importlib.import_module``. + greeting : str + Expected return value from the generated package's ``hello`` function. + + Returns + ------- + None + The helper returns when the generated import and assertion succeed. + + Raises + ------ + AssertionError + Raised by ``pytest-copier`` when the generated command exits + unsuccessfully. + """ script = ( "import importlib; " f"module = importlib.import_module({package!r}); " @@ -43,5 +108,25 @@ def check_generated_import(project: CopierProject, package: str, greeting: str) def read_generated_file(project: CopierProject, relative_path: str) -> str: - """Read a rendered project file as UTF-8 text.""" + """Read a rendered project file as UTF-8 text. + + Parameters + ---------- + project : CopierProject + Rendered ``pytest-copier`` project to read from. + relative_path : str + Path to the generated file, relative to the rendered project root. + + Returns + ------- + str + File contents decoded as UTF-8 text. + + Raises + ------ + FileNotFoundError + Raised when the requested generated file does not exist. + OSError + Raised when the file exists but cannot be read. + """ return (project / relative_path).read_text(encoding="utf-8") diff --git a/tests/helpers/tooling_contracts.py b/tests/helpers/tooling_contracts.py index db506a4..c8bd4b6 100644 --- a/tests/helpers/tooling_contracts.py +++ b/tests/helpers/tooling_contracts.py @@ -12,7 +12,30 @@ def assert_common_make_targets(makefile: str) -> None: - """Assert Makefile targets shared by all generated variants.""" + """Assert Makefile targets shared by all generated variants. + + Parameters + ---------- + makefile : str + UTF-8 text of a generated Makefile. + + Returns + ------- + None + The helper returns after all shared Makefile assertions pass. + + Raises + ------ + AssertionError + Raised when a shared generated Makefile target or cleanup path is + missing. + + Examples + -------- + Validate shared targets after reading a generated Makefile:: + + assert_common_make_targets(makefile) + """ assert "lint-python: build" in makefile, "Makefile should expose lint-python" assert "lint: lint-python" in makefile, "lint should delegate to lint-python" assert ".uv-cache .uv-tools" in makefile, "clean should remove uv state dirs" @@ -31,27 +54,155 @@ def assert_generated_tooling_contracts( pure_wheel_action: str, use_rust: bool, ) -> None: - """Assert generated Python/Rust tooling contracts from one validator.""" - assert_pyproject_contracts( + """Assert generated Python/Rust tooling contracts from one validator. + + Parameters + ---------- + package_name : str + Generated Python import package name. + agents : str + UTF-8 text of the generated ``AGENTS.md`` file. + pyproject : dict[str, Any] + Parsed generated ``pyproject.toml`` mapping. + makefile : str + UTF-8 text of the generated Makefile. + ci_workflow : str + UTF-8 text of the generated CI workflow. + release_workflow : str + UTF-8 text of the generated release workflow. + build_wheels_workflow : str + UTF-8 text of the generated build-wheels workflow. + build_wheels_action : str + UTF-8 text of the generated build-wheels composite action. + pure_wheel_action : str + UTF-8 text of the generated pure-wheel composite action. + use_rust : bool + Whether the rendered variant includes the optional Rust extension. + + Returns + ------- + None + The helper returns after all generated tooling contracts pass. + + Raises + ------ + AssertionError + Raised when any generated tooling, workflow, or packaging contract is + missing or variant-inconsistent. + + Examples + -------- + Validate generated contracts after rendering a project:: + + assert_generated_tooling_contracts( + package_name="example_pkg", + agents=agents, + pyproject=pyproject, + makefile=makefile, + ci_workflow=ci_workflow, + release_workflow=release_workflow, + build_wheels_workflow=build_wheels_workflow, + build_wheels_action=build_wheels_action, + pure_wheel_action=pure_wheel_action, + use_rust=False, + ) + """ + _assert_pyproject_contracts( package_name=package_name, pyproject=pyproject, use_rust=use_rust, ) - assert_agents_contracts(agents) - assert_makefile_contracts(makefile=makefile, use_rust=use_rust) - assert_ci_workflow_contracts(ci_workflow=ci_workflow, use_rust=use_rust) - assert_release_workflow_contracts( + _assert_agents_contracts(agents) + _assert_makefile_contracts(makefile=makefile, use_rust=use_rust) + _assert_ci_workflow_contracts(ci_workflow=ci_workflow, use_rust=use_rust) + _assert_release_workflow_contracts( release_workflow=release_workflow, use_rust=use_rust, ) - assert_wheel_workflow_contracts( + _assert_wheel_workflow_contracts( build_wheels_workflow=build_wheels_workflow, build_wheels_action=build_wheels_action, pure_wheel_action=pure_wheel_action, ) -def assert_pyproject_contracts( +def assert_ci_coverage_action_contract( + *, ci_workflow: str, package_name: str, use_rust: bool +) -> None: + """Assert generated CI coverage inputs used by act validation. + + Parameters + ---------- + ci_workflow : str + UTF-8 text of the generated CI workflow. + package_name : str + Generated Python import package name used to derive the coverage + artefact suffix. + use_rust : bool + Whether the rendered variant includes the optional Rust extension. + + Returns + ------- + None + The helper returns after the coverage action contract matches + expectations. + + Raises + ------ + AssertionError + Raised when checkout credentials, coverage action pinning, coverage + output settings, artefact naming, or Rust manifest inputs are wrong. + + Examples + -------- + Validate a rendered CI workflow's coverage step:: + + assert_ci_coverage_action_contract( + ci_workflow=ci_workflow, + package_name="example_pkg", + use_rust=True, + ) + """ + parsed_ci_workflow = parse_yaml_mapping(ci_workflow, "CI workflow") + jobs = require_mapping(parsed_ci_workflow, "jobs", "CI workflow") + lint_test = require_mapping(jobs, "lint-test", "CI workflow jobs") + steps = require_sequence(lint_test, "steps", "CI lint-test job") + assert _checkout_steps_disable_credentials(steps), ( + "expected CI checkout steps to disable credential persistence" + ) + coverage_steps = [ + step + for step in steps + if isinstance(step, dict) and step.get("name") == "Test and Measure Coverage" + ] + assert len(coverage_steps) == 1, "expected one shared coverage action step" + coverage_step = coverage_steps[0] + assert ( + coverage_step.get("uses") + == "leynos/shared-actions/.github/actions/generate-coverage" + "@d400b079fb6a8fa92f7e7b6c57f3d1c92a4b2d54" + ), "expected CI to use the pinned shared coverage action" + coverage_inputs = require_mapping(coverage_step, "with", "coverage step") + assert coverage_inputs.get("output-path") == "coverage.xml", ( + "expected CI coverage output path to match the act assertion" + ) + assert coverage_inputs.get("format") == "cobertura", ( + "expected CI coverage format to match the CodeScene upload" + ) + assert coverage_inputs.get("artefact-name-suffix") == package_name.replace( + "_", "-" + ), "expected package-specific coverage artefact name suffix" + if use_rust: + assert coverage_inputs.get("cargo-manifest") == "rust_extension/Cargo.toml", ( + "expected Rust variant to pass the extension manifest to coverage" + ) + else: + assert "cargo-manifest" not in coverage_inputs, ( + "expected pure-Python variant to omit Rust coverage inputs" + ) + + +def _assert_pyproject_contracts( *, package_name: str, pyproject: dict[str, Any], use_rust: bool ) -> None: """Assert generated Python packaging contracts.""" @@ -98,7 +249,7 @@ def assert_pyproject_contracts( ) -def assert_agents_contracts(agents: str) -> None: +def _assert_agents_contracts(agents: str) -> None: """Assert generated assistant guidance documents act-enabled testing.""" assert "make test WITH_ACT=1" in agents, ( "expected generated AGENTS.md to document act-enabled test runs" @@ -108,7 +259,7 @@ def assert_agents_contracts(agents: str) -> None: ) -def assert_makefile_contracts(*, makefile: str, use_rust: bool) -> None: +def _assert_makefile_contracts(*, makefile: str, use_rust: bool) -> None: """Assert generated Makefile contracts for both template variants.""" assert_common_make_targets(makefile) assert "WITH_ACT ?= 0" in makefile, ( @@ -154,13 +305,13 @@ def assert_makefile_contracts(*, makefile: str, use_rust: bool) -> None: ) -def assert_ci_workflow_contracts(*, ci_workflow: str, use_rust: bool) -> None: +def _assert_ci_workflow_contracts(*, ci_workflow: str, use_rust: bool) -> None: """Assert generated CI workflow contracts.""" parsed_ci_workflow = parse_yaml_mapping(ci_workflow, "CI workflow") jobs = require_mapping(parsed_ci_workflow, "jobs", "CI workflow") lint_test = require_mapping(jobs, "lint-test", "CI workflow jobs") steps = require_sequence(lint_test, "steps", "CI lint-test job") - assert checkout_steps_disable_credentials(steps), ( + assert _checkout_steps_disable_credentials(steps), ( "expected generated CI workflow to disable checkout credentials" ) assert "make check-fmt" in ci_workflow, ( @@ -197,13 +348,15 @@ def assert_ci_workflow_contracts(*, ci_workflow: str, use_rust: bool) -> None: ) -def assert_release_workflow_contracts(*, release_workflow: str, use_rust: bool) -> None: +def _assert_release_workflow_contracts( + *, release_workflow: str, use_rust: bool +) -> None: """Assert generated release workflow contracts.""" parsed_release_workflow = parse_yaml_mapping(release_workflow, "release workflow") jobs = require_mapping(parsed_release_workflow, "jobs", "release workflow") release = require_mapping(jobs, "release", "release workflow jobs") release_steps = require_sequence(release, "steps", "release job") - assert checkout_steps_disable_credentials(release_steps), ( + assert _checkout_steps_disable_credentials(release_steps), ( "expected generated release workflow to disable checkout credentials" ) assert "softprops/action-gh-release@v2" in release_workflow, ( @@ -241,7 +394,7 @@ def assert_release_workflow_contracts(*, release_workflow: str, use_rust: bool) ) -def assert_wheel_workflow_contracts( +def _assert_wheel_workflow_contracts( *, build_wheels_workflow: str, build_wheels_action: str, @@ -277,7 +430,7 @@ def assert_wheel_workflow_contracts( ) -def checkout_steps_disable_credentials(steps: list[Any]) -> bool: +def _checkout_steps_disable_credentials(steps: list[Any]) -> bool: """Return whether all checkout steps disable credential persistence.""" checkout_steps = [ step diff --git a/tests/test_template.py b/tests/test_template.py index 6832322..aed4a00 100644 --- a/tests/test_template.py +++ b/tests/test_template.py @@ -6,15 +6,6 @@ verify generated files, Make targets, package imports, and Rust-specific output without requiring callers to inspect the rendered project tree manually. -Typical usage is to run this module through pytest after changing template -files, generated Makefile targets, or package layout: - -Examples --------- -Run the generated-template checks directly:: - - python -m pytest tests/test_template.py -v - The tests create temporary projects, install generated dependencies through the rendered ``make all`` target, and may download toolchain packages into the normal user caches used by those generated projects. @@ -30,10 +21,7 @@ from tests.helpers.generated_files import ( parse_toml_file, - parse_yaml_mapping, read_generated_text, - require_mapping, - require_sequence, ) from tests.helpers.rendering import ( check_generated_import, @@ -42,9 +30,9 @@ run_quality_gates, ) from tests.helpers.tooling_contracts import ( + assert_ci_coverage_action_contract, assert_common_make_targets, assert_generated_tooling_contracts, - checkout_steps_disable_credentials, ) @@ -291,7 +279,28 @@ def test_generated_tooling_contracts( package_name: str, use_rust: bool, ) -> None: - """Generated variants expose the expected Python and optional Rust tooling.""" + """Generated variants expose the expected Python and optional Rust tooling. + + Parameters + ---------- + copier + ``pytest-copier`` fixture used to render the template. + tmp_path + Temporary directory where the rendered project is created. + target_dir + Temporary project directory name for the rendered variant. + project_name + Project name answer passed to Copier. + package_name + Package name answer passed to Copier. + use_rust + Whether the rendered variant includes the optional Rust extension. + + Returns + ------- + None + The test passes when the generated tooling contracts are satisfied. + """ project = render_project( tmp_path / target_dir, copier, @@ -349,7 +358,29 @@ def test_generated_github_workflows_match_act_validation_contract( package_name: str, use_rust: bool, ) -> None: - """Rendered workflows expose stable black-box inputs for act validation.""" + """Rendered workflows expose stable black-box inputs for act validation. + + Parameters + ---------- + copier + ``pytest-copier`` fixture used to render the template. + tmp_path + Temporary directory where the rendered project is created. + target_dir + Temporary project directory name for the rendered variant. + project_name + Project name answer passed to Copier. + package_name + Package name answer passed to Copier. + use_rust + Whether the rendered variant includes the optional Rust extension. + + Returns + ------- + None + The test passes when the generated CI coverage action contract matches + the act validation expectations. + """ project = render_project( tmp_path / target_dir, copier, @@ -358,41 +389,8 @@ def test_generated_github_workflows_match_act_validation_contract( use_rust=use_rust, ) ci_workflow = read_generated_text(project / ".github" / "workflows" / "ci.yml") - parsed_ci_workflow = parse_yaml_mapping(ci_workflow, "CI workflow") - jobs = require_mapping(parsed_ci_workflow, "jobs", "CI workflow") - lint_test = require_mapping(jobs, "lint-test", "CI workflow jobs") - steps = require_sequence(lint_test, "steps", "CI lint-test job") - - assert checkout_steps_disable_credentials(steps), ( - "expected CI checkout steps to disable credential persistence" - ) - coverage_steps = [ - step - for step in steps - if isinstance(step, dict) and step.get("name") == "Test and Measure Coverage" - ] - assert len(coverage_steps) == 1, "expected one shared coverage action step" - coverage_step = coverage_steps[0] - assert ( - coverage_step.get("uses") - == "leynos/shared-actions/.github/actions/generate-coverage" - "@d400b079fb6a8fa92f7e7b6c57f3d1c92a4b2d54" - ), "expected CI to use the pinned shared coverage action" - coverage_inputs = require_mapping(coverage_step, "with", "coverage step") - assert coverage_inputs.get("output-path") == "coverage.xml", ( - "expected CI coverage output path to match the act assertion" - ) - assert coverage_inputs.get("format") == "cobertura", ( - "expected CI coverage format to match the CodeScene upload" + assert_ci_coverage_action_contract( + ci_workflow=ci_workflow, + package_name=package_name, + use_rust=use_rust, ) - assert coverage_inputs.get("artefact-name-suffix") == package_name.replace( - "_", "-" - ), "expected package-specific coverage artefact name suffix" - if use_rust: - assert coverage_inputs.get("cargo-manifest") == "rust_extension/Cargo.toml", ( - "expected Rust variant to pass the extension manifest to coverage" - ) - else: - assert "cargo-manifest" not in coverage_inputs, ( - "expected pure-Python variant to omit Rust coverage inputs" - ) From 13ff3b3a05fa63ea968d89b1fae34e87380ab351 Mon Sep 17 00:00:00 2001 From: Payton McIntosh Date: Mon, 1 Jun 2026 14:03:33 +0100 Subject: [PATCH 06/32] Address helper contract review findings Delegate rendered file reads through the shared generated-file reader so OS errors become assertion failures consistently. Document `WITH_ACT` in generated user guidance, document the helper module split for parent-template contributors, and share the parsed CI workflow mapping across tooling contract checks. --- docs/developers-guide.md | 42 ++++++++++++++++++++++++++++++ template/docs/users-guide.md.jinja | 15 +++++++++++ tests/helpers/rendering.py | 10 +++---- tests/helpers/tooling_contracts.py | 32 +++++++++++++++++++---- 4 files changed, 89 insertions(+), 10 deletions(-) diff --git a/docs/developers-guide.md b/docs/developers-guide.md index 07908a0..84e7fe3 100644 --- a/docs/developers-guide.md +++ b/docs/developers-guide.md @@ -50,3 +50,45 @@ Rust documentation output, maturin configuration, and cargo error messaging. The optional `act` tests run generated GitHub Actions workflows as black-box integration checks. Their parser separates structured-log handling from the domain assertions so workflow format details stay local to the adapter helpers. + +### Test Helper Modules + +Test helpers are organised under `tests/helpers/` into three modules, each with +a distinct responsibility: + +**`tests/helpers/rendering.py`** — project rendering and command execution. + +- `render_project` wraps the `pytest-copier` fixture to render a temporary + project with explicit template answers. +- `run_quality_gates` runs the generated `make all` target via the rendered + project wrapper. +- `check_generated_import` imports the generated package through `uv run` and + asserts its `hello()` return value. +- `read_generated_file` reads a file from the rendered project root as UTF-8 + text, converting OS errors into `pytest.fail` exceptions. + +**`tests/helpers/generated_files.py`** — file parsing with assertion-focused +error context. + +- `read_generated_text` reads a `Path` and converts `OSError` into + `pytest.fail`. +- `parse_toml_file` reads and parses a TOML file, converting decode errors + into `pytest.fail`. +- `parse_yaml_mapping` parses a YAML string and asserts the result is a + mapping. +- `require_mapping` / `require_sequence` extract nested keys from a parsed + mapping, failing with a schema-path message when absent or the wrong type. + +**`tests/helpers/tooling_contracts.py`** — generated tooling contract +assertions. + +- `assert_common_make_targets` validates Makefile targets shared by all + generated variants. +- `assert_generated_tooling_contracts` is the single entry-point that + validates packaging configuration, AGENTS.md guidance, Makefile wiring, CI + and release workflow structure, and wheel workflow/action contracts. +- `assert_ci_coverage_action_contract` validates the shared coverage action + step and its inputs, including optional Rust cargo-manifest inputs. + +All public helpers carry NumPy-style docstrings. Internal helpers (prefixed +`_`) are private to the module and not part of the test API. diff --git a/template/docs/users-guide.md.jinja b/template/docs/users-guide.md.jinja index 9070e0c..75e0e7b 100644 --- a/template/docs/users-guide.md.jinja +++ b/template/docs/users-guide.md.jinja @@ -36,6 +36,21 @@ If cargo is missing from the local environment, generated Rust test targets fail early with a clear error instead of falling through to an unusable `cargo` invocation. +## Local GitHub Actions Validation + +The generated Makefile supports optional local workflow validation using +[`act`](https://github.com/nektos/act). When `act` is installed and Docker is +available, pass `WITH_ACT=1` to the `test` target: + +```bash +make test WITH_ACT=1 +``` + +This sets `RUN_ACT_VALIDATION=1` for the pytest invocation, enabling the +act-based integration tests that run the generated CI workflow locally. +Omitting `WITH_ACT` (or setting it to `0`) skips act validation; the rest of +the test suite runs unchanged. + ## Cleaning Local State Run `make clean` to remove local build and cache outputs, including `.venv`, diff --git a/tests/helpers/rendering.py b/tests/helpers/rendering.py index e0f81b2..093e828 100644 --- a/tests/helpers/rendering.py +++ b/tests/helpers/rendering.py @@ -7,6 +7,8 @@ from pytest_copier.plugin import CopierFixture, CopierProject +from tests.helpers.generated_files import read_generated_text + def run_quality_gates(project: CopierProject) -> None: """Run the rendered project's public quality gate. @@ -124,9 +126,7 @@ def read_generated_file(project: CopierProject, relative_path: str) -> str: Raises ------ - FileNotFoundError - Raised when the requested generated file does not exist. - OSError - Raised when the file exists but cannot be read. + pytest.fail.Exception + Raised when the requested generated file cannot be read. """ - return (project / relative_path).read_text(encoding="utf-8") + return read_generated_text(project / relative_path) diff --git a/tests/helpers/tooling_contracts.py b/tests/helpers/tooling_contracts.py index c8bd4b6..ea53c78 100644 --- a/tests/helpers/tooling_contracts.py +++ b/tests/helpers/tooling_contracts.py @@ -107,6 +107,7 @@ def assert_generated_tooling_contracts( use_rust=False, ) """ + parsed_ci_workflow = _parse_ci_workflow(ci_workflow) _assert_pyproject_contracts( package_name=package_name, pyproject=pyproject, @@ -114,7 +115,11 @@ def assert_generated_tooling_contracts( ) _assert_agents_contracts(agents) _assert_makefile_contracts(makefile=makefile, use_rust=use_rust) - _assert_ci_workflow_contracts(ci_workflow=ci_workflow, use_rust=use_rust) + _assert_ci_workflow_contracts( + parsed_ci_workflow=parsed_ci_workflow, + ci_workflow=ci_workflow, + use_rust=use_rust, + ) _assert_release_workflow_contracts( release_workflow=release_workflow, use_rust=use_rust, @@ -163,7 +168,7 @@ def assert_ci_coverage_action_contract( use_rust=True, ) """ - parsed_ci_workflow = parse_yaml_mapping(ci_workflow, "CI workflow") + parsed_ci_workflow = _parse_ci_workflow(ci_workflow) jobs = require_mapping(parsed_ci_workflow, "jobs", "CI workflow") lint_test = require_mapping(jobs, "lint-test", "CI workflow jobs") steps = require_sequence(lint_test, "steps", "CI lint-test job") @@ -202,6 +207,11 @@ def assert_ci_coverage_action_contract( ) +def _parse_ci_workflow(ci_workflow: str) -> dict[str, Any]: + """Parse a generated CI workflow as a YAML mapping.""" + return parse_yaml_mapping(ci_workflow, "CI workflow") + + def _assert_pyproject_contracts( *, package_name: str, pyproject: dict[str, Any], use_rust: bool ) -> None: @@ -305,9 +315,21 @@ def _assert_makefile_contracts(*, makefile: str, use_rust: bool) -> None: ) -def _assert_ci_workflow_contracts(*, ci_workflow: str, use_rust: bool) -> None: - """Assert generated CI workflow contracts.""" - parsed_ci_workflow = parse_yaml_mapping(ci_workflow, "CI workflow") +def _assert_ci_workflow_contracts( + *, parsed_ci_workflow: dict[str, Any], ci_workflow: str, use_rust: bool +) -> None: + """Assert generated CI workflow contracts. + + Parameters + ---------- + parsed_ci_workflow : dict[str, Any] + Parsed generated CI workflow mapping. + ci_workflow : str + UTF-8 text of the generated CI workflow, used for string-level contract + assertions. + use_rust : bool + Whether the rendered variant includes the optional Rust extension. + """ jobs = require_mapping(parsed_ci_workflow, "jobs", "CI workflow") lint_test = require_mapping(jobs, "lint-test", "CI workflow jobs") steps = require_sequence(lint_test, "steps", "CI lint-test job") From 12e52e5444910fe9478ce5ea85e44de53ce43731 Mon Sep 17 00:00:00 2001 From: Payton McIntosh Date: Wed, 3 Jun 2026 10:13:26 +0100 Subject: [PATCH 07/32] Normalize developer guide spelling Use the mandated Oxford `-ize` spelling in the test helper module section of the developer guide. --- docs/developers-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/developers-guide.md b/docs/developers-guide.md index 84e7fe3..293aaa7 100644 --- a/docs/developers-guide.md +++ b/docs/developers-guide.md @@ -53,7 +53,7 @@ domain assertions so workflow format details stay local to the adapter helpers. ### Test Helper Modules -Test helpers are organised under `tests/helpers/` into three modules, each with +Test helpers are organized under `tests/helpers/` into three modules, each with a distinct responsibility: **`tests/helpers/rendering.py`** — project rendering and command execution. From 04e678fcefb5c1696aae15b53a3e0f354c5a086e Mon Sep 17 00:00:00 2001 From: Payton McIntosh Date: Wed, 3 Jun 2026 10:19:42 +0100 Subject: [PATCH 08/32] Add direct helper error-path tests Cover generated-file parsing failures, rendered-file read errors, and public tooling contract edge cases without rendering full Copier projects. These tests make helper fallibility contracts explicit for OSError, invalid TOML, invalid YAML, wrong schema shapes, Makefile target checks, and CI coverage action requirements. --- tests/test_helpers.py | 292 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 292 insertions(+) create mode 100644 tests/test_helpers.py diff --git a/tests/test_helpers.py b/tests/test_helpers.py new file mode 100644 index 0000000..d39d2df --- /dev/null +++ b/tests/test_helpers.py @@ -0,0 +1,292 @@ +"""Validate direct helper-module error handling and edge cases. + +The tests in this module exercise support helpers without rendering a full +Copier project. They keep helper fallibility contracts explicit by checking +``pytest.fail`` conversion paths, generated-file schema helpers, and tooling +contract assertions directly. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, cast + +import pytest +from pytest_copier.plugin import CopierProject + +from tests.helpers.generated_files import ( + parse_toml_file, + parse_yaml_mapping, + read_generated_text, + require_mapping, + require_sequence, +) +from tests.helpers.rendering import read_generated_file +from tests.helpers.tooling_contracts import ( + assert_ci_coverage_action_contract, + assert_common_make_targets, +) + + +def test_read_generated_text_converts_os_errors( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Convert generated-file read errors into pytest failures. + + Parameters + ---------- + monkeypatch + Pytest monkeypatch fixture used to force ``Path.read_text`` to raise an + ``OSError``. + + Returns + ------- + None + The test passes when the helper raises ``pytest.fail.Exception`` with + path context instead of propagating the raw OS error. + """ + + def raise_os_error(self: Path, *, encoding: str | None = None) -> str: + """Raise a deterministic OS error for helper error-path coverage.""" + raise OSError("cannot read fixture") + + monkeypatch.setattr(Path, "read_text", raise_os_error) + + with pytest.raises(pytest.fail.Exception, match="could not read generated file"): + read_generated_text(Path("generated.txt")) + + +def test_parse_toml_file_reports_decode_errors(tmp_path: Path) -> None: + """Convert generated TOML decode errors into pytest failures. + + Parameters + ---------- + tmp_path + Temporary directory used to write invalid TOML content. + + Returns + ------- + None + The test passes when invalid TOML raises ``pytest.fail.Exception`` with + generated-file context. + """ + pyproject = tmp_path / "pyproject.toml" + pyproject.write_text("[project\nname = 'broken'\n", encoding="utf-8") + + with pytest.raises(pytest.fail.Exception, match="could not parse generated TOML"): + parse_toml_file(pyproject) + + +def test_parse_yaml_mapping_reports_invalid_yaml() -> None: + """Convert generated YAML parser errors into pytest failures. + + Parameters + ---------- + None + This test does not use pytest fixtures. + + Returns + ------- + None + The test passes when invalid YAML raises ``pytest.fail.Exception`` with + the supplied label. + """ + with pytest.raises(pytest.fail.Exception, match="could not parse generated CI"): + parse_yaml_mapping("jobs: [unterminated", "CI") + + +def test_parse_yaml_mapping_requires_mapping_root() -> None: + """Reject generated YAML documents that do not parse to mappings. + + Parameters + ---------- + None + This test does not use pytest fixtures. + + Returns + ------- + None + The test passes when a sequence root raises ``pytest.fail.Exception``. + """ + with pytest.raises( + pytest.fail.Exception, + match="expected generated CI workflow to parse as a mapping", + ): + parse_yaml_mapping("- lint\n- test\n", "CI workflow") + + +def test_generated_file_schema_helpers_require_expected_shapes() -> None: + """Fail with schema-path context for wrong nested value shapes. + + Parameters + ---------- + None + This test does not use pytest fixtures. + + Returns + ------- + None + The test passes when mapping and sequence helpers accept valid values + and fail on missing or incorrectly typed values. + """ + mapping: dict[str, Any] = { + "jobs": {"lint-test": {}}, + "steps": [{"name": "Check"}], + } + + assert require_mapping(mapping, "jobs", "CI workflow") == {"lint-test": {}} + assert require_sequence(mapping, "steps", "CI lint-test job") == [{"name": "Check"}] + + with pytest.raises( + pytest.fail.Exception, + match="expected CI workflow to include mapping key 'jobs'", + ): + require_mapping({"jobs": []}, "jobs", "CI workflow") + + with pytest.raises( + pytest.fail.Exception, + match="expected CI lint-test job to include sequence key 'steps'", + ): + require_sequence({"steps": {}}, "steps", "CI lint-test job") + + +def test_read_generated_file_uses_shared_error_contract(tmp_path: Path) -> None: + """Read rendered files through the shared generated-file helper contract. + + Parameters + ---------- + tmp_path + Temporary rendered-project stand-in used as the project root. + + Returns + ------- + None + The test passes when existing files are read and missing files raise + ``pytest.fail.Exception`` instead of raw filesystem exceptions. + """ + generated = tmp_path / "docs" / "users-guide.md" + generated.parent.mkdir() + generated.write_text("generated docs\n", encoding="utf-8") + project = cast(CopierProject, tmp_path) + + assert read_generated_file(project, "docs/users-guide.md") == "generated docs\n" + with pytest.raises(pytest.fail.Exception, match="could not read generated file"): + read_generated_file(project, "missing.md") + + +def test_common_make_targets_reports_missing_contracts() -> None: + """Report missing shared Makefile targets through assertion messages. + + Parameters + ---------- + None + This test does not use pytest fixtures. + + Returns + ------- + None + The test passes when the shared target assertion accepts a complete + Makefile fragment and rejects a fragment missing required targets. + """ + assert_common_make_targets( + "lint-python: build\nlint: lint-python\nclean:\n\trm -rf .uv-cache .uv-tools\n" + ) + + with pytest.raises(AssertionError, match="Makefile should expose lint-python"): + assert_common_make_targets("lint: lint-python\n") + + +def test_ci_coverage_action_contract_validates_pure_python_edges() -> None: + """Validate pure-Python CI coverage action edge cases. + + Parameters + ---------- + None + This test does not use pytest fixtures. + + Returns + ------- + None + The test passes when a valid pure-Python workflow is accepted and a + workflow with persistent checkout credentials is rejected. + """ + assert_ci_coverage_action_contract( + ci_workflow=_ci_workflow( + persist_credentials="false", + coverage_inputs=" artefact-name-suffix: helper-pkg\n", + ), + package_name="helper_pkg", + use_rust=False, + ) + + with pytest.raises( + AssertionError, + match="expected CI checkout steps to disable credential persistence", + ): + assert_ci_coverage_action_contract( + ci_workflow=_ci_workflow( + persist_credentials="true", + coverage_inputs=" artefact-name-suffix: helper-pkg\n", + ), + package_name="helper_pkg", + use_rust=False, + ) + + +def test_ci_coverage_action_contract_validates_rust_manifest_edge() -> None: + """Validate Rust CI coverage action cargo-manifest requirements. + + Parameters + ---------- + None + This test does not use pytest fixtures. + + Returns + ------- + None + The test passes when the Rust workflow requires the extension + ``cargo-manifest`` input on the shared coverage action. + """ + with pytest.raises( + AssertionError, + match="expected Rust variant to pass the extension manifest to coverage", + ): + assert_ci_coverage_action_contract( + ci_workflow=_ci_workflow( + persist_credentials="false", + coverage_inputs=" artefact-name-suffix: helper-pkg\n", + ), + package_name="helper_pkg", + use_rust=True, + ) + + assert_ci_coverage_action_contract( + ci_workflow=_ci_workflow( + persist_credentials="false", + coverage_inputs=( + " artefact-name-suffix: helper-pkg\n" + " cargo-manifest: rust_extension/Cargo.toml\n" + ), + ), + package_name="helper_pkg", + use_rust=True, + ) + + +def _ci_workflow(*, persist_credentials: str, coverage_inputs: str) -> str: + """Return a minimal generated CI workflow for coverage-contract tests.""" + return f"""\ +name: CI +jobs: + lint-test: + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: {persist_credentials} + - name: Test and Measure Coverage + uses: leynos/shared-actions/.github/actions/generate-coverage@d400b079fb6a8fa92f7e7b6c57f3d1c92a4b2d54 + with: + output-path: coverage.xml + format: cobertura +{coverage_inputs}\ +""" From 0afdfb85f1f1b06a622096153b349e452667b6e3 Mon Sep 17 00:00:00 2001 From: Payton McIntosh Date: Wed, 3 Jun 2026 12:14:12 +0100 Subject: [PATCH 09/32] Tighten helper unit test contracts Avoid monkey-patching `Path.read_text` when testing generated-file read errors, defer the `CopierProject` import to type checking, and add explicit messages to direct equality assertions. --- tests/test_helpers.py | 48 +++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/tests/test_helpers.py b/tests/test_helpers.py index d39d2df..d306fe0 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -9,10 +9,9 @@ from __future__ import annotations from pathlib import Path -from typing import Any, cast +from typing import TYPE_CHECKING, Any, cast import pytest -from pytest_copier.plugin import CopierProject from tests.helpers.generated_files import ( parse_toml_file, @@ -27,33 +26,27 @@ assert_common_make_targets, ) +if TYPE_CHECKING: + from pytest_copier.plugin import CopierProject -def test_read_generated_text_converts_os_errors( - monkeypatch: pytest.MonkeyPatch, -) -> None: + +def test_read_generated_text_converts_os_errors() -> None: """Convert generated-file read errors into pytest failures. Parameters ---------- - monkeypatch - Pytest monkeypatch fixture used to force ``Path.read_text`` to raise an - ``OSError``. + None + This test passes a nonexistent ``Path`` to ``read_generated_text``. Returns ------- None - The test passes when the helper raises ``pytest.fail.Exception`` with - path context instead of propagating the raw OS error. + The test passes when ``read_generated_text`` raises + ``pytest.fail.Exception`` with path context instead of propagating the + raw ``FileNotFoundError`` from ``Path.read_text``. """ - - def raise_os_error(self: Path, *, encoding: str | None = None) -> str: - """Raise a deterministic OS error for helper error-path coverage.""" - raise OSError("cannot read fixture") - - monkeypatch.setattr(Path, "read_text", raise_os_error) - with pytest.raises(pytest.fail.Exception, match="could not read generated file"): - read_generated_text(Path("generated.txt")) + read_generated_text(Path("nonexistent_generated.txt")) def test_parse_toml_file_reports_decode_errors(tmp_path: Path) -> None: @@ -134,8 +127,16 @@ def test_generated_file_schema_helpers_require_expected_shapes() -> None: "steps": [{"name": "Check"}], } - assert require_mapping(mapping, "jobs", "CI workflow") == {"lint-test": {}} - assert require_sequence(mapping, "steps", "CI lint-test job") == [{"name": "Check"}] + assert require_mapping(mapping, "jobs", "CI workflow") == {"lint-test": {}}, ( + "expected require_mapping(mapping, 'jobs', 'CI workflow') to return a " + "jobs mapping containing lint-test" + ) + assert require_sequence(mapping, "steps", "CI lint-test job") == [ + {"name": "Check"} + ], ( + "expected require_sequence(mapping, 'steps', 'CI lint-test job') to " + "return steps sequence [{'name': 'Check'}]" + ) with pytest.raises( pytest.fail.Exception, @@ -167,9 +168,12 @@ def test_read_generated_file_uses_shared_error_contract(tmp_path: Path) -> None: generated = tmp_path / "docs" / "users-guide.md" generated.parent.mkdir() generated.write_text("generated docs\n", encoding="utf-8") - project = cast(CopierProject, tmp_path) + project = cast("CopierProject", tmp_path) - assert read_generated_file(project, "docs/users-guide.md") == "generated docs\n" + assert read_generated_file(project, "docs/users-guide.md") == "generated docs\n", ( + "expected read_generated_file(project, 'docs/users-guide.md') to return " + "the generated docs text" + ) with pytest.raises(pytest.fail.Exception, match="could not read generated file"): read_generated_file(project, "missing.md") From c4e02b2506875978a4ef18e54d5af27aa766aaf0 Mon Sep 17 00:00:00 2001 From: Payton McIntosh Date: Wed, 3 Jun 2026 12:25:18 +0100 Subject: [PATCH 10/32] Cover parent Makefile and helper module roles Expand helper module docstrings to describe their relationships within the test helper layer. Use a tmp_path-scoped missing file for generated-file read error coverage and add direct tests for the parent Makefile help and test target contracts. --- tests/helpers/generated_files.py | 10 +++- tests/helpers/rendering.py | 11 ++++- tests/helpers/tooling_contracts.py | 11 ++++- tests/test_helpers.py | 74 ++++++++++++++++++++++++++++-- 4 files changed, 99 insertions(+), 7 deletions(-) diff --git a/tests/helpers/generated_files.py b/tests/helpers/generated_files.py index 744644a..18eb585 100644 --- a/tests/helpers/generated_files.py +++ b/tests/helpers/generated_files.py @@ -1,4 +1,12 @@ -"""Parse and validate generated project files in template tests.""" +"""Parse generated project files for template contract tests. + +This module owns assertion-focused file reading and structured data parsing for +rendered Copier projects. Rendering helpers use ``read_generated_text`` so +filesystem errors become pytest failures consistently, while tooling contract +helpers consume ``parse_yaml_mapping``, ``require_mapping``, and +``require_sequence`` to keep workflow schema checks readable. Keep raw +generated-file I/O here so template tests share one error-reporting boundary. +""" from __future__ import annotations diff --git a/tests/helpers/rendering.py b/tests/helpers/rendering.py index 093e828..2409931 100644 --- a/tests/helpers/rendering.py +++ b/tests/helpers/rendering.py @@ -1,4 +1,13 @@ -"""Render generated projects and run their public commands in tests.""" +"""Render Copier projects and bridge generated-file helper APIs. + +This module wraps ``pytest-copier`` interactions used by template tests: +rendering projects, running generated quality gates, importing generated +packages, and reading rendered files relative to a project root. File reads +delegate to :mod:`tests.helpers.generated_files` so rendering-oriented tests +share the same pytest failure semantics as lower-level generated-file parsers. +Tooling contract tests call these helpers before passing rendered text into +:mod:`tests.helpers.tooling_contracts`. +""" from __future__ import annotations diff --git a/tests/helpers/tooling_contracts.py b/tests/helpers/tooling_contracts.py index ea53c78..304a229 100644 --- a/tests/helpers/tooling_contracts.py +++ b/tests/helpers/tooling_contracts.py @@ -1,4 +1,13 @@ -"""Assertion helpers for generated tooling and workflow contracts.""" +"""Assert rendered tooling contracts for generated project variants. + +This module contains the higher-level contract checks used after +``tests.helpers.rendering`` renders a project and +``tests.helpers.generated_files`` parses its structured files. The public +helpers validate generated Makefile targets, packaging metadata, assistant +guidance, and GitHub workflow structure for both pure-Python and Python/Rust +variants. Keep workflow and tooling assertions here so top-level template +tests stay focused on scenario setup. +""" from __future__ import annotations diff --git a/tests/test_helpers.py b/tests/test_helpers.py index d306fe0..fcaa89c 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -8,6 +8,7 @@ from __future__ import annotations +import subprocess from pathlib import Path from typing import TYPE_CHECKING, Any, cast @@ -30,13 +31,14 @@ from pytest_copier.plugin import CopierProject -def test_read_generated_text_converts_os_errors() -> None: +def test_read_generated_text_converts_os_errors(tmp_path: Path) -> None: """Convert generated-file read errors into pytest failures. Parameters ---------- - None - This test passes a nonexistent ``Path`` to ``read_generated_text``. + tmp_path + Temporary directory used to build a missing file path for + ``read_generated_text``. Returns ------- @@ -45,8 +47,10 @@ def test_read_generated_text_converts_os_errors() -> None: ``pytest.fail.Exception`` with path context instead of propagating the raw ``FileNotFoundError`` from ``Path.read_text``. """ + missing_path = tmp_path / "nonexistent_generated.txt" + with pytest.raises(pytest.fail.Exception, match="could not read generated file"): - read_generated_text(Path("nonexistent_generated.txt")) + read_generated_text(missing_path) def test_parse_toml_file_reports_decode_errors(tmp_path: Path) -> None: @@ -277,6 +281,68 @@ def test_ci_coverage_action_contract_validates_rust_manifest_edge() -> None: ) +def test_parent_makefile_help_target_lists_available_targets() -> None: + """Validate the parent repository ``help`` target output. + + Parameters + ---------- + None + This test does not use pytest fixtures. + + Returns + ------- + None + The test passes when ``make help`` advertises the parent ``help`` and + ``test`` targets. + """ + result = subprocess.run( + ["make", "help"], + check=True, + capture_output=True, + encoding="utf-8", + ) + + assert "Available targets:" in result.stdout, ( + "expected parent Makefile help target to print an available-targets header" + ) + assert "help" in result.stdout, ( + "expected parent Makefile help target to list the help target" + ) + assert "test" in result.stdout, ( + "expected parent Makefile help target to list the test target" + ) + + +def test_parent_makefile_test_target_uses_requisite_pytest_command() -> None: + """Validate the parent repository ``test`` target command contract. + + Parameters + ---------- + None + This test does not use pytest fixtures. + + Returns + ------- + None + The test passes when the parent Makefile exposes ``test`` as phony and + runs pytest through ``uvx`` with the required template-test packages. + """ + makefile = Path("Makefile").read_text(encoding="utf-8") + + assert ".PHONY: help test" in makefile, ( + "expected parent Makefile to mark help and test as phony targets" + ) + assert "test: ## Run template tests" in makefile, ( + "expected parent Makefile to expose a documented test target" + ) + assert ( + "uvx --with pytest-copier --with pyyaml --with syrupy pytest tests/" in makefile + ), ( + "expected parent Makefile test target to run pytest through uvx with " + "pytest-copier, pyyaml, and syrupy" + ) + + def _ci_workflow(*, persist_credentials: str, coverage_inputs: str) -> str: """Return a minimal generated CI workflow for coverage-contract tests.""" return f"""\ From e47324d78f6d2b3695580700c4001cf92484ceb3 Mon Sep 17 00:00:00 2001 From: Payton McIntosh Date: Wed, 3 Jun 2026 14:11:52 +0100 Subject: [PATCH 11/32] Anchor Makefile tests and strengthen snapshots Resolve parent Makefile tests relative to the repository root so they do not depend on pytest's invocation directory. Add semantic assertions beside snapshot checks for generated help targets and the rendered pure module's `hello() -> str` function contract. --- tests/test_helpers.py | 6 +++++- tests/test_template.py | 30 ++++++++++++++++++++++++++++-- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/tests/test_helpers.py b/tests/test_helpers.py index fcaa89c..f381186 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -31,6 +31,9 @@ from pytest_copier.plugin import CopierProject +REPO_ROOT = Path(__file__).resolve().parent.parent + + def test_read_generated_text_converts_os_errors(tmp_path: Path) -> None: """Convert generated-file read errors into pytest failures. @@ -299,6 +302,7 @@ def test_parent_makefile_help_target_lists_available_targets() -> None: ["make", "help"], check=True, capture_output=True, + cwd=REPO_ROOT, encoding="utf-8", ) @@ -327,7 +331,7 @@ def test_parent_makefile_test_target_uses_requisite_pytest_command() -> None: The test passes when the parent Makefile exposes ``test`` as phony and runs pytest through ``uvx`` with the required template-test packages. """ - makefile = Path("Makefile").read_text(encoding="utf-8") + makefile = (REPO_ROOT / "Makefile").read_text(encoding="utf-8") assert ".PHONY: help test" in makefile, ( "expected parent Makefile to mark help and test as phony targets" diff --git a/tests/test_template.py b/tests/test_template.py index aed4a00..665c8dc 100644 --- a/tests/test_template.py +++ b/tests/test_template.py @@ -13,6 +13,7 @@ from __future__ import annotations +import ast from pathlib import Path import pytest @@ -76,7 +77,12 @@ def test_python_only_help_output_snapshot( use_rust=False, ) - assert project.run("make help") == snapshot + help_output = project.run("make help") + for target in ["build", "check-fmt", "lint", "typecheck", "test", "help"]: + assert f" {target}" in help_output, ( + f"expected generated help output to list the {target!r} target" + ) + assert help_output == snapshot @pytest.mark.parametrize( @@ -126,7 +132,27 @@ def test_pure_module_snapshot( use_rust=use_rust, ) - assert read_generated_file(project, f"{package_name}/pure.py") == snapshot + pure_module = read_generated_file(project, f"{package_name}/pure.py") + parsed_module = ast.parse(pure_module) + hello_functions = [ + node + for node in parsed_module.body + if isinstance(node, ast.FunctionDef) and node.name == "hello" + ] + assert len(hello_functions) == 1, ( + "expected generated pure.py to define exactly one hello function" + ) + hello_function = hello_functions[0] + assert not hello_function.args.args, ( + "expected generated pure.py hello function to accept no positional arguments" + ) + assert isinstance(hello_function.returns, ast.Name), ( + "expected generated pure.py hello function to declare a return annotation" + ) + assert hello_function.returns.id == "str", ( + "expected generated pure.py hello function to return str" + ) + assert pure_module == snapshot def test_python_only_template(copier: CopierFixture, tmp_path: Path) -> None: From 38beb8b90ee4220f5e5ca78132aded53bacdf2f9 Mon Sep 17 00:00:00 2001 From: Payton McIntosh Date: Wed, 3 Jun 2026 15:29:57 +0100 Subject: [PATCH 12/32] Document snapshot quality guidance Expand generated `syrupy` snapshot guidance with criteria for meaningful, focused snapshots, semantic assertions, nondeterministic field normalization, and brittleness prevention. Keep Rust `insta` guidance aligned with the same policy and assert the rendered AGENTS.md contract in template tests. --- template/AGENTS.md.jinja | 16 +++++++++++++--- tests/helpers/tooling_contracts.py | 15 +++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/template/AGENTS.md.jinja b/template/AGENTS.md.jinja index 16ef690..a1ac82a 100644 --- a/template/AGENTS.md.jinja +++ b/template/AGENTS.md.jinja @@ -131,7 +131,16 @@ - For Python work, use `pytest` for unit tests and `pytest-bdd` for behavioural tests. Cover happy paths, unhappy paths, and relevant edge cases. - Snapshot tests (using `syrupy`) should be provided where multivariant output - format consistency is relevant to the requirements. + format consistency is relevant to the requirements. Snapshot tests must + capture meaningful, reviewer-useful contracts rather than generic dumps of + broad objects. Keep snapshots focused on stable output boundaries, pair them + with semantic assertions for the behaviours they represent, and avoid + snapshot-only coverage for logic that can be asserted directly. Redact or + normalize nondeterministic fields such as timestamps, absolute paths, random + identifiers, ordering-dependent maps, hostnames, and environment-specific + values before snapshotting. Do not accept brittle snapshots that churn after + harmless formatting or dependency changes; narrow the captured output until a + snapshot failure identifies a real contract change. - Add end-to-end tests where a change affects externally observable workflows, integration contracts, persistence, command-line behaviour, network boundaries, user interface flows, or other system-level behaviour. @@ -186,8 +195,9 @@ project: - Ensure that new features are validated with unit and behavioural tests before release using `rstest` and `rstest-bdd`. Cover happy paths, unhappy paths, and relevant edge cases. -- For Rust: snapshot tests (using `insta`) should be provided where multivariant - output format consistency is relevant to the requirements. +- For Rust: snapshot tests (using `insta`) should follow the same snapshot + quality criteria and should be provided where multivariant output format + consistency is relevant to the requirements. - Add end-to-end tests where a change affects externally observable workflows, integration contracts, persistence, command-line behaviour, network boundaries, user interface flows, or other system-level behaviour. diff --git a/tests/helpers/tooling_contracts.py b/tests/helpers/tooling_contracts.py index 304a229..0f27d8f 100644 --- a/tests/helpers/tooling_contracts.py +++ b/tests/helpers/tooling_contracts.py @@ -276,6 +276,21 @@ def _assert_agents_contracts(agents: str) -> None: assert "RUN_ACT_VALIDATION=1" in agents, ( "expected generated AGENTS.md to describe the pytest act environment" ) + assert "meaningful, reviewer-useful contracts" in agents, ( + "expected generated AGENTS.md to require meaningful snapshot contracts" + ) + assert "generic dumps" in agents, ( + "expected generated AGENTS.md to warn against generic snapshots" + ) + assert "semantic assertions" in agents, ( + "expected generated AGENTS.md to pair snapshots with semantic assertions" + ) + assert "normalize nondeterministic fields" in agents, ( + "expected generated AGENTS.md to require nondeterministic field redaction" + ) + assert "brittle snapshots" in agents, ( + "expected generated AGENTS.md to warn against brittle snapshot churn" + ) def _assert_makefile_contracts(*, makefile: str, use_rust: bool) -> None: From 8e1cda71e582b6cf7dd978e39283846186eebc19 Mon Sep 17 00:00:00 2001 From: Payton McIntosh Date: Wed, 3 Jun 2026 15:54:39 +0100 Subject: [PATCH 13/32] Validate AGENTS make target command contracts Use `make-parser` in rendered tooling contract tests to compare the make targets documented in generated `AGENTS.md` with generated Makefile rules. Document the Python and Rust command flags that those make targets pass through and assert the parsed Makefile recipes include the stated flags. --- Makefile | 2 +- template/AGENTS.md.jinja | 32 ++++-- tests/helpers/tooling_contracts.py | 169 +++++++++++++++++++++++++++++ tests/test_helpers.py | 5 +- 4 files changed, 196 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index ccde514..5bba46b 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ MAKEFLAGS += --no-print-directory test: ## Run template tests - uvx --with pytest-copier --with pyyaml --with syrupy pytest tests/ + uvx --with pytest-copier --with pyyaml --with syrupy --with make-parser pytest tests/ help: ## Show available targets @grep -E '^[a-zA-Z_-]+:.*?##' $(MAKEFILE_LIST) | \ diff --git a/template/AGENTS.md.jinja b/template/AGENTS.md.jinja index a1ac82a..54e629a 100644 --- a/template/AGENTS.md.jinja +++ b/template/AGENTS.md.jinja @@ -76,6 +76,15 @@ - **Formatting:** Adheres to formatting standards (`make check-fmt`; use `make fmt` to apply fixes). - **Typechecking:** Passes type checking (`make typecheck`). + - The generated Makefile wiring for these targets is: + - `make check-fmt` runs Ruff formatting checks with + `ruff format --check $(PYTHON_TARGETS)`. + - `make lint` runs `make lint-python`; `make lint-python` runs + `ruff check $(PYTHON_TARGETS)` and the PyPy-backed Pylint runner against + `$(PYLINT_TARGETS)`. + - `make typecheck` runs `ty check $(PYTHON_TARGETS)`. + - `make test` runs `pytest -v -n $(PYTEST_XDIST_WORKERS)` and honours + `WITH_ACT=1` through `RUN_ACT_VALIDATION=1`. {% if use_rust -%} - For Rust files: - **Testing:** Passes relevant unit and behavioural tests (`make test`). @@ -163,27 +172,32 @@ project: - `make check-fmt` executes: ```sh - cargo fmt --workspace -- --check + cargo fmt --manifest-path rust_extension/Cargo.toml --all -- --check ``` - validating formatting across the entire workspace without modifying files. + validating formatting for the Rust extension without modifying files. - `make lint` executes: ```sh - cargo clippy --workspace --all-targets --all-features -- -D warnings + cargo doc --no-deps --manifest-path rust_extension/Cargo.toml + cargo clippy --manifest-path rust_extension/Cargo.toml --all-targets --all-features -- -D warnings + whitaker --all -- --all-targets --all-features ``` - linting every target with all features enabled and denying all Clippy - warnings. + documenting and linting every Rust extension target with all features + enabled and warnings denied. - `make test` executes: ```sh - cargo test --workspace + cargo nextest run --manifest-path rust_extension/Cargo.toml --all-targets --all-features + # or, when cargo-nextest is unavailable: + cargo test --manifest-path rust_extension/Cargo.toml --all-targets --all-features + cargo test --doc --manifest-path rust_extension/Cargo.toml --all-features ``` - running the full workspace test suite. Use `make fmt` - (`cargo fmt --workspace`) to apply formatting fixes reported by the - formatter check. + running the Rust extension test suite and documentation tests. Use + `make fmt` (`cargo fmt --manifest-path rust_extension/Cargo.toml --all`) to + apply formatting fixes reported by the formatter check. - Clippy warnings MUST be disallowed. - Fix any warnings emitted during tests in code instead of silencing them. - Where a function is too long, extract meaningfully named helper functions diff --git a/tests/helpers/tooling_contracts.py b/tests/helpers/tooling_contracts.py index 0f27d8f..e0862c1 100644 --- a/tests/helpers/tooling_contracts.py +++ b/tests/helpers/tooling_contracts.py @@ -11,8 +11,13 @@ from __future__ import annotations +import re +import tempfile +from pathlib import Path from typing import Any +import make_parser + from tests.helpers.generated_files import ( parse_yaml_mapping, require_mapping, @@ -123,6 +128,12 @@ def assert_generated_tooling_contracts( use_rust=use_rust, ) _assert_agents_contracts(agents) + _assert_agents_make_targets_mirror_makefile( + agents=agents, + makefile=makefile, + package_name=package_name, + use_rust=use_rust, + ) _assert_makefile_contracts(makefile=makefile, use_rust=use_rust) _assert_ci_workflow_contracts( parsed_ci_workflow=parsed_ci_workflow, @@ -293,6 +304,164 @@ def _assert_agents_contracts(agents: str) -> None: ) +def _assert_agents_make_targets_mirror_makefile( + *, agents: str, makefile: str, package_name: str, use_rust: bool +) -> None: + """Assert AGENTS.md make target references match parsed Makefile commands.""" + documented_targets = _documented_make_targets(agents) + makefile_rules = _parse_makefile_rules(makefile) + makefile_targets = set(makefile_rules) + missing_targets = sorted(documented_targets - makefile_targets) + assert not missing_targets, ( + "expected every make target documented in generated AGENTS.md to exist " + f"in the generated Makefile, missing: {missing_targets}" + ) + required_documented_targets = { + "check-fmt", + "fmt", + "lint", + "markdownlint", + "nixie", + "test", + "typecheck", + } + missing_documented_targets = sorted( + required_documented_targets - documented_targets + ) + assert not missing_documented_targets, ( + "expected generated AGENTS.md to document the generated Makefile quality " + f"gate targets, missing: {missing_documented_targets}" + ) + _assert_documented_command_flags( + agents=agents, + makefile_rules=makefile_rules, + package_name=package_name, + use_rust=use_rust, + ) + + +def _assert_documented_command_flags( + *, + agents: str, + makefile_rules: dict[str, list[str]], + package_name: str, + use_rust: bool, +) -> None: + """Assert documented make command flags are present in parsed recipes.""" + python_targets = f"{package_name} tests" + command_contracts = { + "check-fmt": [ + ("AGENTS.md", "ruff format --check $(PYTHON_TARGETS)"), + ("Makefile", f"ruff format --check {python_targets}"), + ], + "lint-python": [ + ("AGENTS.md", "ruff check $(PYTHON_TARGETS)"), + ("Makefile", f"ruff check {python_targets}"), + ], + "typecheck": [ + ("AGENTS.md", "ty check $(PYTHON_TARGETS)"), + ("Makefile", f"ty check {python_targets}"), + ], + "test": [ + ("AGENTS.md", "pytest -v -n $(PYTEST_XDIST_WORKERS)"), + ("Makefile", "pytest -v -n auto"), + ], + } + if use_rust: + command_contracts.update( + { + "check-fmt": [ + *command_contracts["check-fmt"], + ( + "AGENTS.md", + "cargo fmt --manifest-path rust_extension/Cargo.toml", + ), + ("AGENTS.md", "--all -- --check"), + ("Makefile", "fmt --manifest-path rust_extension/Cargo.toml"), + ("Makefile", "--all -- --check"), + ], + "lint-rust": [ + ("AGENTS.md", "cargo doc --no-deps"), + ( + "AGENTS.md", + "cargo clippy --manifest-path rust_extension/Cargo.toml", + ), + ("AGENTS.md", "--all-targets --all-features -- -D warnings"), + ("AGENTS.md", "whitaker --all -- --all-targets --all-features"), + ("Makefile", "doc --no-deps"), + ("Makefile", "clippy --manifest-path rust_extension/Cargo.toml"), + ("Makefile", "--all-targets --all-features -- -D warnings"), + ("Makefile", "--all -- --all-targets --all-features"), + ], + "test": [ + *command_contracts["test"], + ( + "AGENTS.md", + "cargo nextest run --manifest-path rust_extension/Cargo.toml", + ), + ( + "AGENTS.md", + "cargo test --manifest-path rust_extension/Cargo.toml", + ), + ( + "AGENTS.md", + "cargo test --doc --manifest-path rust_extension/Cargo.toml", + ), + ("Makefile", "--manifest-path rust_extension/Cargo.toml"), + ("Makefile", "--all-targets --all-features"), + ( + "Makefile", + "test --doc --manifest-path rust_extension/Cargo.toml", + ), + ], + } + ) + for target, checks in command_contracts.items(): + commands = "\n".join(makefile_rules[target]) + for source, fragment in checks: + haystack = agents if source == "AGENTS.md" else commands + assert fragment in haystack, ( + f"expected generated {source} {target!r} command contract to " + f"include {fragment!r}" + ) + + +def _documented_make_targets(agents: str) -> set[str]: + """Return make targets referenced in rendered AGENTS.md guidance.""" + return set(re.findall(r"`make ([a-zA-Z][a-zA-Z_-]*)", agents)) + + +def _parse_makefile_rules(makefile: str) -> dict[str, list[str]]: + """Return generated Makefile rules parsed through make-parser.""" + target_names = set( + re.findall(r"^([a-zA-Z][a-zA-Z_-]*):", makefile, flags=re.MULTILINE) + ) + normalised_targets = { + target: target.replace("-", "_") for target in target_names if "-" in target + } + + def normalise_target(match: re.Match[str]) -> str: + target = match.group(1) + return normalised_targets.get(target, target) + ":" + + normalised_makefile = re.sub( + r"^([a-zA-Z][a-zA-Z_-]*):", + normalise_target, + makefile.replace("?=", "="), + flags=re.MULTILINE, + ) + with tempfile.TemporaryDirectory() as tmp_dir: + makefile_path = Path(tmp_dir) / "Makefile" + makefile_path.write_text(normalised_makefile, encoding="utf-8") + parsed = make_parser.make_load(makefile_path) + normalised_rules = parsed["rules"] + return { + target: normalised_rules[normalised_targets.get(target, target)]["commands"] + for target in target_names + if normalised_targets.get(target, target) in normalised_rules + } + + def _assert_makefile_contracts(*, makefile: str, use_rust: bool) -> None: """Assert generated Makefile contracts for both template variants.""" assert_common_make_targets(makefile) diff --git a/tests/test_helpers.py b/tests/test_helpers.py index f381186..a1ba3b0 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -340,10 +340,11 @@ def test_parent_makefile_test_target_uses_requisite_pytest_command() -> None: "expected parent Makefile to expose a documented test target" ) assert ( - "uvx --with pytest-copier --with pyyaml --with syrupy pytest tests/" in makefile + "uvx --with pytest-copier --with pyyaml --with syrupy --with make-parser " + "pytest tests/" in makefile ), ( "expected parent Makefile test target to run pytest through uvx with " - "pytest-copier, pyyaml, and syrupy" + "pytest-copier, pyyaml, syrupy, and make-parser" ) From ed0e2d3c875278654308572a6c6f229d2882ff31 Mon Sep 17 00:00:00 2001 From: Payton McIntosh Date: Wed, 3 Jun 2026 16:21:03 +0100 Subject: [PATCH 14/32] Guard parent test target uvx lookup Resolve `uvx` before running the parent template test target and fail early with installation guidance when it is unavailable. Update the direct parent Makefile contract test to require the lookup, the helpful error message, and the resolved `$(UV)` test invocation. --- Makefile | 8 +++++++- tests/test_helpers.py | 15 +++++++++++---- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 5bba46b..8ebc2e7 100644 --- a/Makefile +++ b/Makefile @@ -2,8 +2,14 @@ MAKEFLAGS += --no-print-directory +UV := $(shell command -v uvx 2>/dev/null) + +ifeq ($(strip $(UV)),) +$(error uvx is required to run template tests. Install uv from https://docs.astral.sh/uv/getting-started/installation/) +endif + test: ## Run template tests - uvx --with pytest-copier --with pyyaml --with syrupy --with make-parser pytest tests/ + $(UV) --with pytest-copier --with pyyaml --with syrupy --with make-parser pytest tests/ help: ## Show available targets @grep -E '^[a-zA-Z_-]+:.*?##' $(MAKEFILE_LIST) | \ diff --git a/tests/test_helpers.py b/tests/test_helpers.py index a1ba3b0..c3a0103 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -328,8 +328,9 @@ def test_parent_makefile_test_target_uses_requisite_pytest_command() -> None: Returns ------- None - The test passes when the parent Makefile exposes ``test`` as phony and - runs pytest through ``uvx`` with the required template-test packages. + The test passes when the parent Makefile exposes ``test`` as phony, + checks for ``uvx``, and runs pytest through the resolved executable with + the required template-test packages. """ makefile = (REPO_ROOT / "Makefile").read_text(encoding="utf-8") @@ -339,11 +340,17 @@ def test_parent_makefile_test_target_uses_requisite_pytest_command() -> None: assert "test: ## Run template tests" in makefile, ( "expected parent Makefile to expose a documented test target" ) + assert "UV := $(shell command -v uvx 2>/dev/null)" in makefile, ( + "expected parent Makefile to resolve uvx before running tests" + ) + assert "uvx is required to run template tests" in makefile, ( + "expected parent Makefile to fail early with a uvx installation message" + ) assert ( - "uvx --with pytest-copier --with pyyaml --with syrupy --with make-parser " + "$(UV) --with pytest-copier --with pyyaml --with syrupy --with make-parser " "pytest tests/" in makefile ), ( - "expected parent Makefile test target to run pytest through uvx with " + "expected parent Makefile test target to run pytest through $(UV) with " "pytest-copier, pyyaml, syrupy, and make-parser" ) From 8577d0a13e13575a219d909b53c099ce97c19885 Mon Sep 17 00:00:00 2001 From: Payton McIntosh Date: Wed, 3 Jun 2026 16:26:46 +0100 Subject: [PATCH 15/32] Document parent Makefile help target Mention `make help` in the README running-tests guidance so template developers can discover the parent Makefile targets. Also keep the documented parent test dependencies aligned with the current `make-parser`-backed test target. --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 149c882..6b8f6ec 100644 --- a/README.md +++ b/README.md @@ -14,9 +14,10 @@ The test suite relies on the `pytest-copier` plugin and renders generated projects that run Ruff, Pylint via a PyPy-backed runner, `ty`, pytest, and, when the Rust extension is enabled, Clippy, Whitaker, and nextest-aware Rust tests. -Run the parent template tests through the repository `Makefile`. The `test` -target uses `uvx` to provide `pytest-copier`, `PyYAML`, and `syrupy` without a -manually managed virtual environment: +Run the parent template tests through the repository `Makefile`. Run +`make help` to list the available parent Makefile targets. The `test` target +uses `uvx` to provide `pytest-copier`, `PyYAML`, `syrupy`, and `make-parser` +without a manually managed virtual environment: ```bash make test From 1cfa739ead9d4123c89e7039e144641e628340a4 Mon Sep 17 00:00:00 2001 From: Payton McIntosh Date: Wed, 3 Jun 2026 16:47:16 +0100 Subject: [PATCH 16/32] Prove Rust test target runs doctests --- tests/test_template.py | 45 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/tests/test_template.py b/tests/test_template.py index 665c8dc..8c45a94 100644 --- a/tests/test_template.py +++ b/tests/test_template.py @@ -14,6 +14,8 @@ from __future__ import annotations import ast +import shutil +import subprocess from pathlib import Path import pytest @@ -251,6 +253,49 @@ def test_rust_template(copier: CopierFixture, tmp_path: Path) -> None: check_generated_import(proj, "rust_pkg", "hello from Rust") +def test_rust_template_make_test_runs_doctests( + copier: CopierFixture, tmp_path: Path +) -> None: + """Validate that Rust-enabled generated projects gate doctests.""" + proj = copier.copy( + tmp_path / "rust-doctest", + project_name="RustDoctest", + package_name="rust_doctest_pkg", + use_rust=True, + ) + lib_rs = proj / "rust_extension" / "src" / "lib.rs" + lib_rs.write_text( + lib_rs.read_text(encoding="utf-8") + + """ + +/// Deliberately broken doctest used by the parent template regression test. +/// +/// ``` +/// let status = std::process::ExitCode::SUCCESS; +/// assert!(status.success()); +/// ``` +pub fn doctest_regression_marker() {} +""", + encoding="utf-8", + ) + make = shutil.which("make") + assert make is not None, "expected make to be available for generated tests" + + result = subprocess.run( + [make, "test"], + cwd=proj.path, + check=False, + capture_output=True, + text=True, + ) + + output = f"{result.stdout}\n{result.stderr}" + assert result.returncode != 0, "expected make test to fail on broken doctests" + assert "no method named `success`" in output, ( + "expected make test to compile doctests, exposing the broken example" + ) + + def test_rust_template_custom_package(copier: CopierFixture, tmp_path: Path) -> None: """Ensure templating uses the provided package name. From d193267cf6bdf86a785323bc5b92221b71929300 Mon Sep 17 00:00:00 2001 From: Payton McIntosh Date: Wed, 3 Jun 2026 19:06:50 +0100 Subject: [PATCH 17/32] Tighten doctest regression review fixes Expand the Rust doctest regression test docstring to the public NumPy-style shape used by the template tests, align README dependency spelling with the parent Makefile's `pyyaml` token, and include `make-parser` in the optional setup script used for parent test dependencies. --- README.md | 2 +- scripts/setup_test_deps.sh | 2 +- tests/test_template.py | 27 ++++++++++++++++++++++++++- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6b8f6ec..e35a40b 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ the Rust extension is enabled, Clippy, Whitaker, and nextest-aware Rust tests. Run the parent template tests through the repository `Makefile`. Run `make help` to list the available parent Makefile targets. The `test` target -uses `uvx` to provide `pytest-copier`, `PyYAML`, `syrupy`, and `make-parser` +uses `uvx` to provide `pytest-copier`, `pyyaml`, `syrupy`, and `make-parser` without a manually managed virtual environment: ```bash diff --git a/scripts/setup_test_deps.sh b/scripts/setup_test_deps.sh index de84522..ba77007 100755 --- a/scripts/setup_test_deps.sh +++ b/scripts/setup_test_deps.sh @@ -3,4 +3,4 @@ # Generated projects install their own linting and type-checking dependencies. set -euo pipefail -pip install pytest-copier PyYAML syrupy +pip install pytest-copier pyyaml syrupy make-parser diff --git a/tests/test_template.py b/tests/test_template.py index 8c45a94..1c9f64f 100644 --- a/tests/test_template.py +++ b/tests/test_template.py @@ -256,7 +256,32 @@ def test_rust_template(copier: CopierFixture, tmp_path: Path) -> None: def test_rust_template_make_test_runs_doctests( copier: CopierFixture, tmp_path: Path ) -> None: - """Validate that Rust-enabled generated projects gate doctests.""" + """Validate that Rust-enabled generated projects gate doctests. + + Parameters + ---------- + copier : CopierFixture + Fixture used to render the template into a temporary project. + tmp_path : Path + Temporary directory used as the generated project root. + + Returns + ------- + None + The test fails via assertions when the generated ``make test`` target + does not run Rust documentation tests. + + Raises + ------ + None + Expected failures are captured through pytest assertions. + + Notes + ----- + The test injects a deliberately broken Rust doctest and verifies that the + generated project's public ``make test`` target reports the doctest + failure. + """ proj = copier.copy( tmp_path / "rust-doctest", project_name="RustDoctest", From 7be16d88a898f6e9d05a8fa2c6d78bbf7ad46f72 Mon Sep 17 00:00:00 2001 From: Payton McIntosh Date: Wed, 3 Jun 2026 19:37:46 +0100 Subject: [PATCH 18/32] Add generated dependency audit gate Add a generated `make audit` target for Python dependencies with `pip-audit`, and extend Rust-enabled renders with a `rust-audit` target that runs `cargo audit` in the Rust extension crate. Wire CI to run the audit gate and install `cargo-audit` for Rust-enabled projects. Document the new target and add rendered tests for pure-Python and Python/Rust variants using fake tool binaries so parent tests do not contact external vulnerability databases. --- docs/developers-guide.md | 7 +- template/.github/workflows/ci.yml.jinja | 5 + template/AGENTS.md.jinja | 12 +++ template/Makefile.jinja | 14 ++- template/docs/developers-guide.md | 8 +- template/docs/users-guide.md.jinja | 8 ++ template/pyproject.toml.jinja | 1 + tests/__snapshots__/test_template.ambr | 1 + tests/helpers/tooling_contracts.py | 38 ++++++- tests/test_audit.py | 135 ++++++++++++++++++++++++ tests/test_helpers.py | 6 +- tests/test_template.py | 2 +- 12 files changed, 229 insertions(+), 8 deletions(-) create mode 100644 tests/test_audit.py diff --git a/docs/developers-guide.md b/docs/developers-guide.md index 293aaa7..93462ef 100644 --- a/docs/developers-guide.md +++ b/docs/developers-guide.md @@ -8,6 +8,9 @@ projects and run their public quality gates. `template/Makefile.jinja` defines the generated developer workflow. The default `all` target runs build, formatting, linting, typechecking, and tests. +The generated `audit` target is an explicit security gate run by CI. It runs +`pip-audit` for every rendered project and, when `use_rust` is enabled, also +runs `cargo audit` in the Rust extension crate. The generated lint targets are split by language: @@ -15,6 +18,8 @@ The generated lint targets are split by language: - `lint-rust` exists only when `use_rust` is enabled and runs rustdoc, Clippy, and Whitaker. - `lint` delegates to the applicable language-specific targets. +- `audit` exists for both generated variants and runs `pip-audit`; Rust-enabled + variants delegate to `rust-audit` for `cargo audit`. Tool revisions are exposed as Makefile variables such as `PYLINT_PYPY_SHIM_REF` and `WHITAKER_INSTALLER_REV`, so generated projects can @@ -24,7 +29,7 @@ override pins without editing target recipes. `template/.github/workflows/ci.yml.jinja` mirrors the generated local gates. It sets up Python, optionally sets up Rust, runs `make check-fmt`, `make lint`, -and `make typecheck`, then delegates coverage to +`make typecheck`, and `make audit`, then delegates coverage to `leynos/shared-actions/.github/actions/generate-coverage`. Rust-enabled workflows pass `rust_extension/Cargo.toml` to the coverage action diff --git a/template/.github/workflows/ci.yml.jinja b/template/.github/workflows/ci.yml.jinja index 6c751ea..6c057de 100644 --- a/template/.github/workflows/ci.yml.jinja +++ b/template/.github/workflows/ci.yml.jinja @@ -41,6 +41,7 @@ jobs: with: cache-directories: | ~/.cargo/bin/cargo-nextest + ~/.cargo/bin/cargo-audit ~/.cargo/bin/whitaker-installer ~/.local/bin/whitaker ~/.local/share/whitaker @@ -52,6 +53,7 @@ jobs: --rev "${WHITAKER_INSTALLER_REV}" \ whitaker-installer whitaker-installer --cranelift + cargo install --locked cargo-audit {% endif %} - name: Install CLI tools @@ -74,6 +76,9 @@ jobs: - name: Run typechecker run: make typecheck + - name: Audit dependencies + run: make audit + - name: Test and Measure Coverage uses: leynos/shared-actions/.github/actions/generate-coverage@d400b079fb6a8fa92f7e7b6c57f3d1c92a4b2d54 with: diff --git a/template/AGENTS.md.jinja b/template/AGENTS.md.jinja index 54e629a..a71f9d8 100644 --- a/template/AGENTS.md.jinja +++ b/template/AGENTS.md.jinja @@ -76,6 +76,7 @@ - **Formatting:** Adheres to formatting standards (`make check-fmt`; use `make fmt` to apply fixes). - **Typechecking:** Passes type checking (`make typecheck`). + - **Auditing:** Passes dependency vulnerability checks (`make audit`). - The generated Makefile wiring for these targets is: - `make check-fmt` runs Ruff formatting checks with `ruff format --check $(PYTHON_TARGETS)`. @@ -85,12 +86,14 @@ - `make typecheck` runs `ty check $(PYTHON_TARGETS)`. - `make test` runs `pytest -v -n $(PYTEST_XDIST_WORKERS)` and honours `WITH_ACT=1` through `RUN_ACT_VALIDATION=1`. + - `make audit` runs `pip-audit`. {% if use_rust -%} - For Rust files: - **Testing:** Passes relevant unit and behavioural tests (`make test`). - **Linting:** Passes lint checks (`make lint`). - **Formatting:** Adheres to formatting standards (`make check-fmt`; use `make fmt` to apply fixes). + - **Auditing:** Passes Rust dependency vulnerability checks (`make audit`). {% endif %} - **Markdown files (`.md` only):** - **Linting:** Passes markdown lint checks (`make markdownlint`). @@ -198,6 +201,15 @@ project: running the Rust extension test suite and documentation tests. Use `make fmt` (`cargo fmt --manifest-path rust_extension/Cargo.toml --all`) to apply formatting fixes reported by the formatter check. + - `make audit` executes: + + ```sh + pip-audit + cargo audit + ``` + + checking Python dependencies and Rust extension dependencies for known + vulnerabilities. - Clippy warnings MUST be disallowed. - Fix any warnings emitted during tests in code instead of silencing them. - Where a function is too long, extract meaningfully named helper functions diff --git a/template/Makefile.jinja b/template/Makefile.jinja index 9bc76cc..0da5f70 100644 --- a/template/Makefile.jinja +++ b/template/Makefile.jinja @@ -36,8 +36,8 @@ WHITAKER_INSTALLER_REV ?= f768c2e53c47df13658af1168a67851d388750bf WHITAKER ?= $(or $(shell command -v whitaker 2>/dev/null),$(wildcard $(USER_WHITAKER)),whitaker) {% endif %} -.PHONY: help all clean build build-release lint lint-python fmt check-fmt \ - markdownlint nixie test typecheck $(TOOLS) $(VENV_TOOLS){% if use_rust %} lint-rust whitaker{% endif %} +.PHONY: help all audit clean build build-release lint lint-python fmt check-fmt \ + markdownlint nixie test typecheck $(TOOLS) $(VENV_TOOLS){% if use_rust %} lint-rust rust-audit whitaker{% endif %} .DEFAULT_GOAL := all @@ -150,6 +150,16 @@ typecheck: build ## Run typechecking $(UV_ENV) $(UV) run ty --version $(UV_ENV) $(UV) run ty check $(PYTHON_TARGETS) +audit: build ## Audit dependencies for known vulnerabilities + $(UV_ENV) $(UV) run pip-audit +{% if use_rust %} + $(MAKE) rust-audit + +rust-audit: ## Audit Rust extension dependencies for known vulnerabilities + $(call ensure_cargo) + cd $(RUST_CRATE_DIR) && $(CARGO) audit +{% endif %} + markdownlint: $(MDLINT) ## Lint Markdown files env -u NO_COLOR $(MDLINT) '**/*.md' diff --git a/template/docs/developers-guide.md b/template/docs/developers-guide.md index 3b84a63..65f5279 100644 --- a/template/docs/developers-guide.md +++ b/template/docs/developers-guide.md @@ -9,6 +9,10 @@ The public entrypoint for formatting, linting, typechecking, and tests is failure, and changes should be reconciled with the aggregate gate before being considered complete. +Run `make audit` as the dependency vulnerability gate. It runs `pip-audit` for +Python dependencies, and Rust-enabled projects also run `cargo audit` from the +`rust_extension` crate directory. + ## Automation scripts The [Scripting standards](scripting-standards.md) document provides guidance for @@ -28,8 +32,8 @@ actions under `.github/`. - `.github/workflows/ci.yml` runs on pushes to `main` and on pull requests. It sets up Python 3.13, installs `uv`, validates the generated `Makefile` with - `mbake`, runs `make build`, `make check-fmt`, `make lint`, and - `make typecheck`, then delegates coverage generation to the shared coverage + `mbake`, runs `make build`, `make check-fmt`, `make lint`, `make typecheck`, + and `make audit`, then delegates coverage generation to the shared coverage action. When the Rust extension is enabled, it also sets up Rust, installs Rust lint and test tools, and passes `rust_extension/Cargo.toml` to coverage. - `.github/workflows/release.yml` publishes wheels when a `v*.*.*` tag is diff --git a/template/docs/users-guide.md.jinja b/template/docs/users-guide.md.jinja index 75e0e7b..c73d131 100644 --- a/template/docs/users-guide.md.jinja +++ b/template/docs/users-guide.md.jinja @@ -12,6 +12,7 @@ these targets in order: - `lint`: run `lint-python` and, when Rust is enabled, `lint-rust`. - `typecheck`: run `ty check`. - `test`: run pytest and, when Rust is enabled, Rust tests. +- `audit`: run `pip-audit` and, when Rust is enabled, `cargo audit`. The `lint-python` target runs Ruff followed by Pylint via a PyPy-backed runner. The Pylint runner is installed through `uv tool run` from the pinned @@ -26,6 +27,13 @@ When the Rust extension is enabled, `lint-rust` runs: The generated Makefile installs Whitaker on demand before local Rust linting when it is not already available. +## Dependency Auditing + +Run `make audit` to check generated project dependencies for known +vulnerabilities. All generated projects run `pip-audit` against the Python +environment created by `uv sync --group dev`. Rust-enabled projects also run +`cargo audit` from the `rust_extension` crate directory. + ## Rust Test Behaviour Rust-enabled projects use `cargo nextest run` when `cargo-nextest` is available. diff --git a/template/pyproject.toml.jinja b/template/pyproject.toml.jinja index c2a8c20..c7477bc 100644 --- a/template/pyproject.toml.jinja +++ b/template/pyproject.toml.jinja @@ -10,6 +10,7 @@ dependencies = [] [dependency-groups] dev = [ "pytest", + "pip-audit", "ruff", "pyright", "ty", diff --git a/tests/__snapshots__/test_template.ambr b/tests/__snapshots__/test_template.ambr index 3224ee8..eee44c2 100644 --- a/tests/__snapshots__/test_template.ambr +++ b/tests/__snapshots__/test_template.ambr @@ -53,6 +53,7 @@ lint Run linters lint-python Run Python linters typecheck Run typechecking + audit Audit dependencies for known vulnerabilities markdownlint Lint Markdown files nixie Validate Mermaid diagrams test Run tests diff --git a/tests/helpers/tooling_contracts.py b/tests/helpers/tooling_contracts.py index e0862c1..3ee7cba 100644 --- a/tests/helpers/tooling_contracts.py +++ b/tests/helpers/tooling_contracts.py @@ -52,6 +52,7 @@ def assert_common_make_targets(makefile: str) -> None: """ assert "lint-python: build" in makefile, "Makefile should expose lint-python" assert "lint: lint-python" in makefile, "lint should delegate to lint-python" + assert "audit: build" in makefile, "Makefile should expose audit" assert ".uv-cache .uv-tools" in makefile, "clean should remove uv state dirs" @@ -250,7 +251,7 @@ def _assert_pyproject_contracts( assert isinstance(dev_dependencies, list), ( "expected generated pyproject.toml to include a dev dependency group" ) - for dependency in ["pytest", "ruff", "pyright", "ty", "pytest-xdist"]: + for dependency in ["pytest", "pip-audit", "ruff", "pyright", "ty", "pytest-xdist"]: assert dependency in dev_dependencies, ( f"expected generated dev dependencies to include {dependency}" ) @@ -317,6 +318,7 @@ def _assert_agents_make_targets_mirror_makefile( f"in the generated Makefile, missing: {missing_targets}" ) required_documented_targets = { + "audit", "check-fmt", "fmt", "lint", @@ -362,6 +364,10 @@ def _assert_documented_command_flags( ("AGENTS.md", "ty check $(PYTHON_TARGETS)"), ("Makefile", f"ty check {python_targets}"), ], + "audit": [ + ("AGENTS.md", "pip-audit"), + ("Makefile", "pip-audit"), + ], "test": [ ("AGENTS.md", "pytest -v -n $(PYTEST_XDIST_WORKERS)"), ("Makefile", "pytest -v -n auto"), @@ -414,6 +420,15 @@ def _assert_documented_command_flags( "test --doc --manifest-path rust_extension/Cargo.toml", ), ], + "audit": [ + *command_contracts["audit"], + ("AGENTS.md", "cargo audit"), + ("Makefile", "$(MAKE) rust-audit"), + ], + "rust-audit": [ + ("Makefile", "cd rust_extension"), + ("Makefile", "cargo) audit"), + ], } ) for target, checks in command_contracts.items(): @@ -486,6 +501,9 @@ def _assert_makefile_contracts(*, makefile: str, use_rust: bool) -> None: assert "test: build $(VENV_TOOLS)" in makefile, ( "expected generated Makefile test target to depend on the project env" ) + assert "$(UV_ENV) $(UV) run pip-audit" in makefile, ( + "expected generated audit target to run pip-audit" + ) if use_rust: assert "TEST_CMD :=" in makefile, ( "expected Rust variant to select nextest or cargo test" @@ -499,6 +517,12 @@ def _assert_makefile_contracts(*, makefile: str, use_rust: bool) -> None: assert "$(CARGO) $(TEST_CMD) $(TEST_FLAGS)" in makefile, ( "expected Rust variant tests to use the selected cargo test command" ) + assert "rust-audit:" in makefile, ( + "expected Rust variant to expose the rust-audit target" + ) + assert "cd $(RUST_CRATE_DIR) && $(CARGO) audit" in makefile, ( + "expected Rust variant audit target to run cargo audit" + ) else: assert "lint-rust" not in makefile, ( "expected pure-Python variant to omit Rust lint targets" @@ -506,6 +530,9 @@ def _assert_makefile_contracts(*, makefile: str, use_rust: bool) -> None: assert "TEST_CMD :=" not in makefile, ( "expected pure-Python variant to omit Rust test command selection" ) + assert "rust-audit" not in makefile, ( + "expected pure-Python variant to omit Rust audit targets" + ) def _assert_ci_workflow_contracts( @@ -538,6 +565,9 @@ def _assert_ci_workflow_contracts( assert "make typecheck" in ci_workflow, ( "expected generated CI workflow to run the typecheck gate" ) + assert "make audit" in ci_workflow, ( + "expected generated CI workflow to run the dependency audit gate" + ) assert "make build" in ci_workflow, ( "expected generated CI workflow to build the project before checks" ) @@ -554,6 +584,9 @@ def _assert_ci_workflow_contracts( assert "cargo-manifest: rust_extension/Cargo.toml" in ci_workflow, ( "expected Rust variant CI to pass the Rust manifest to coverage" ) + assert "cargo install --locked cargo-audit" in ci_workflow, ( + "expected Rust variant CI to install cargo-audit" + ) else: assert "setup-rust" not in ci_workflow, ( "expected pure-Python CI to omit Rust setup" @@ -561,6 +594,9 @@ def _assert_ci_workflow_contracts( assert "cargo-manifest" not in ci_workflow, ( "expected pure-Python CI to omit Rust coverage inputs" ) + assert "cargo-audit" not in ci_workflow, ( + "expected pure-Python CI to omit Rust audit installation" + ) def _assert_release_workflow_contracts( diff --git a/tests/test_audit.py b/tests/test_audit.py new file mode 100644 index 0000000..c229ea7 --- /dev/null +++ b/tests/test_audit.py @@ -0,0 +1,135 @@ +"""Validate rendered dependency audit Makefile targets. + +This module exercises the generated ``make audit`` target for pure-Python and +Python/Rust renders without contacting external vulnerability databases. The +tests use fake ``uv`` and ``cargo`` executables to record the audit commands +that the rendered Makefile dispatches. +""" + +from __future__ import annotations + +import shutil +import subprocess +from pathlib import Path + +import pytest +from pytest_copier.plugin import CopierFixture + +from tests.helpers.rendering import render_project + + +@pytest.mark.parametrize( + ("target_dir", "project_name", "package_name", "use_rust"), + [ + ("audit-pure", "AuditPure", "audit_pure", False), + ("audit-rust", "AuditRust", "audit_rust", True), + ], +) +def test_generated_audit_target_runs_expected_tools( + copier: CopierFixture, + tmp_path: Path, + target_dir: str, + project_name: str, + package_name: str, + use_rust: bool, +) -> None: + """Validate generated audit target command dispatch. + + Parameters + ---------- + copier : CopierFixture + Fixture used to render the template into a temporary project. + tmp_path : Path + Temporary directory used for the rendered project and fake tools. + target_dir : str + Temporary project directory name for the rendered variant. + project_name : str + Project name answer passed to Copier. + package_name : str + Package name answer passed to Copier. + use_rust : bool + Whether the rendered variant includes the optional Rust extension. + + Returns + ------- + None + The test passes when ``make audit`` runs ``pip-audit`` for every + variant and runs ``cargo audit`` only for Rust-enabled projects. + """ + project = render_project( + tmp_path / target_dir, + copier, + project_name=project_name, + package_name=package_name, + use_rust=use_rust, + ) + command_log = tmp_path / f"{target_dir}-audit.log" + fake_uv = _write_fake_uv(tmp_path / f"{target_dir}-bin", command_log) + fake_cargo = _write_fake_cargo(tmp_path / f"{target_dir}-cargo", command_log) + make = shutil.which("make") + assert make is not None, "expected make to be available for generated tests" + + result = subprocess.run( + [make, "audit", f"UV={fake_uv}", f"CARGO={fake_cargo}"], + cwd=project.path, + check=False, + capture_output=True, + text=True, + ) + + output = f"{result.stdout}\n{result.stderr}" + assert result.returncode == 0, output + log_lines = command_log.read_text(encoding="utf-8").splitlines() + assert "uv|pip-audit" in log_lines, ( + "expected generated audit target to run pip-audit through uv" + ) + if use_rust: + assert f"cargo|{project.path / 'rust_extension'}|audit" in log_lines, ( + "expected Rust-enabled audit target to run cargo audit in rust_extension" + ) + else: + assert all(not line.startswith("cargo|") for line in log_lines), ( + "expected pure-Python audit target to omit cargo audit" + ) + + +def _write_fake_uv(bin_dir: Path, command_log: Path) -> Path: + """Write a fake uv executable that records audit invocations.""" + bin_dir.mkdir(parents=True, exist_ok=True) + uv_path = bin_dir / "uv" + uv_path.write_text( + "#!/usr/bin/env sh\n" + 'if [ "$1" = venv ]; then\n' + " mkdir -p .venv/bin\n" + " exit 0\n" + "fi\n" + 'if [ "$1" = sync ]; then\n' + " exit 0\n" + "fi\n" + 'if [ "$1" = run ]; then\n' + " shift\n" + f" printf 'uv|%s\\n' \"$*\" >> '{command_log}'\n" + " exit 0\n" + "fi\n" + "exit 0\n", + encoding="utf-8", + ) + uv_path.chmod(0o755) + return uv_path + + +def _write_fake_cargo(bin_dir: Path, command_log: Path) -> Path: + """Write a fake cargo executable that records audit invocations.""" + bin_dir.mkdir(parents=True, exist_ok=True) + cargo_path = bin_dir / "cargo" + cargo_path.write_text( + "#!/usr/bin/env sh\n" + 'if [ "$1" = audit ]; then\n' + f" printf 'cargo|%s|%s\\n' \"$PWD\" \"$*\" >> '{command_log}'\n" + " exit 0\n" + "fi\n" + "exit 0\n", + encoding="utf-8", + ) + cargo_path.chmod(0o755) + return cargo_path diff --git a/tests/test_helpers.py b/tests/test_helpers.py index c3a0103..bb81004 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -200,7 +200,11 @@ def test_common_make_targets_reports_missing_contracts() -> None: Makefile fragment and rejects a fragment missing required targets. """ assert_common_make_targets( - "lint-python: build\nlint: lint-python\nclean:\n\trm -rf .uv-cache .uv-tools\n" + "lint-python: build\n" + "lint: lint-python\n" + "audit: build\n" + "clean:\n" + "\trm -rf .uv-cache .uv-tools\n" ) with pytest.raises(AssertionError, match="Makefile should expose lint-python"): diff --git a/tests/test_template.py b/tests/test_template.py index 1c9f64f..756dfb1 100644 --- a/tests/test_template.py +++ b/tests/test_template.py @@ -80,7 +80,7 @@ def test_python_only_help_output_snapshot( ) help_output = project.run("make help") - for target in ["build", "check-fmt", "lint", "typecheck", "test", "help"]: + for target in ["build", "check-fmt", "lint", "typecheck", "audit", "test", "help"]: assert f" {target}" in help_output, ( f"expected generated help output to list the {target!r} target" ) From dd96c24f04cbf9233b4ef13e81c697edbf1a7dd3 Mon Sep 17 00:00:00 2001 From: Payton McIntosh Date: Wed, 3 Jun 2026 19:58:36 +0100 Subject: [PATCH 19/32] Split act validation into separate workflows Add parent and rendered act-validation workflows that run `make test WITH_ACT=1` separately from the normal CI and coverage paths. Keep the parent `CI` workflow on the normal `make test` gate so rendered workflow validation does not hold up the standard template test workflow. Wire the parent Makefile to pass `WITH_ACT` through to pytest and extend the workflow contract tests to enforce the split. --- .github/workflows/act-validation.yml | 44 +++++++++++++++++ .github/workflows/ci.yml | 37 ++++++++++++++ Makefile | 4 +- README.md | 5 ++ docs/developers-guide.md | 4 ++ .../workflows/act-validation.yml.jinja | 43 ++++++++++++++++ template/docs/developers-guide.md | 3 ++ tests/helpers/tooling_contracts.py | 41 ++++++++++++++++ tests/test_helpers.py | 12 +++-- tests/test_parent_ci.py | 49 +++++++++++++++++++ tests/test_template.py | 4 ++ 11 files changed, 242 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/act-validation.yml create mode 100644 .github/workflows/ci.yml create mode 100644 template/.github/workflows/act-validation.yml.jinja create mode 100644 tests/test_parent_ci.py diff --git a/.github/workflows/act-validation.yml b/.github/workflows/act-validation.yml new file mode 100644 index 0000000..cbeb338 --- /dev/null +++ b/.github/workflows/act-validation.yml @@ -0,0 +1,44 @@ +name: Act Validation + +on: + push: + branches: [main] + pull_request: + +jobs: + act-validation: + runs-on: ubuntu-latest + env: + ACT_VERSION: v0.2.80 + steps: + - name: Check out repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: '3.13' + + - name: Install uv + uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 + + - name: Set up Rust + uses: leynos/shared-actions/.github/actions/setup-rust@d400b079fb6a8fa92f7e7b6c57f3d1c92a4b2d54 + with: + toolchain: stable + + - name: Install template test tools + run: | + curl -fsSL "https://github.com/nektos/act/releases/download/${ACT_VERSION}/act_Linux_x86_64.tar.gz" | + sudo tar -xz -C /usr/local/bin act + npm install -g markdownlint-cli2 + uv tool install mbake + uv tool install mdformat-all + + - name: Check Docker runtime + run: docker info + + - name: Run template tests with act validation + run: make test WITH_ACT=1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..df006f0 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,37 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: '3.13' + + - name: Install uv + uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 + + - name: Set up Rust + uses: leynos/shared-actions/.github/actions/setup-rust@d400b079fb6a8fa92f7e7b6c57f3d1c92a4b2d54 + with: + toolchain: stable + + - name: Install template test tools + run: | + npm install -g markdownlint-cli2 + uv tool install mbake + uv tool install mdformat-all + + - name: Run template tests + run: make test diff --git a/Makefile b/Makefile index 8ebc2e7..df34ed6 100644 --- a/Makefile +++ b/Makefile @@ -3,13 +3,15 @@ MAKEFLAGS += --no-print-directory UV := $(shell command -v uvx 2>/dev/null) +WITH_ACT ?= 0 +ACT_TEST_ENV = $(if $(filter 1 true yes on,$(WITH_ACT)),RUN_ACT_VALIDATION=1,) ifeq ($(strip $(UV)),) $(error uvx is required to run template tests. Install uv from https://docs.astral.sh/uv/getting-started/installation/) endif test: ## Run template tests - $(UV) --with pytest-copier --with pyyaml --with syrupy --with make-parser pytest tests/ + $(ACT_TEST_ENV) $(UV) --with pytest-copier --with pyyaml --with syrupy --with make-parser pytest tests/ help: ## Show available targets @grep -E '^[a-zA-Z_-]+:.*?##' $(MAKEFILE_LIST) | \ diff --git a/README.md b/README.md index e35a40b..e780bdc 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,11 @@ without a manually managed virtual environment: make test ``` +Run `make test WITH_ACT=1` to include the act-backed workflow integration tests +when `act` and Docker are available. Parent repository CI runs this mode in a +separate act-validation workflow so rendered in-template GitHub workflows do not +hold up the normal template test workflow. + Generated projects install and run their own tooling, including Ruff, Pylint via PyPy, `ty`, pytest, and, when Rust is enabled, Clippy, Whitaker, and nextest. diff --git a/docs/developers-guide.md b/docs/developers-guide.md index 93462ef..87154a0 100644 --- a/docs/developers-guide.md +++ b/docs/developers-guide.md @@ -32,6 +32,10 @@ sets up Python, optionally sets up Rust, runs `make check-fmt`, `make lint`, `make typecheck`, and `make audit`, then delegates coverage to `leynos/shared-actions/.github/actions/generate-coverage`. +`template/.github/workflows/act-validation.yml.jinja` keeps rendered workflow +validation separate from generated application CI. It installs `act`, verifies +Docker availability, and runs `make test WITH_ACT=1`. + Rust-enabled workflows pass `rust_extension/Cargo.toml` to the coverage action because the generated Python project root does not contain a Rust manifest. diff --git a/template/.github/workflows/act-validation.yml.jinja b/template/.github/workflows/act-validation.yml.jinja new file mode 100644 index 0000000..20c4bc5 --- /dev/null +++ b/template/.github/workflows/act-validation.yml.jinja @@ -0,0 +1,43 @@ +name: Act Validation + +on: + push: + branches: [main] + pull_request: + +jobs: + act-validation: + runs-on: ubuntu-latest + env: + ACT_VERSION: v0.2.80 + steps: + - name: Check out repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: '3.13' + + - name: Install uv + uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 + + {% if use_rust %} + - name: Set up Rust + uses: leynos/shared-actions/.github/actions/setup-rust@d400b079fb6a8fa92f7e7b6c57f3d1c92a4b2d54 + with: + toolchain: stable + + {% endif %} + - name: Install act + run: | + curl -fsSL "https://github.com/nektos/act/releases/download/${ACT_VERSION}/act_Linux_x86_64.tar.gz" | + sudo tar -xz -C /usr/local/bin act + + - name: Check Docker runtime + run: docker info + + - name: Run tests with act validation + run: make test WITH_ACT=1 diff --git a/template/docs/developers-guide.md b/template/docs/developers-guide.md index 65f5279..5723a94 100644 --- a/template/docs/developers-guide.md +++ b/template/docs/developers-guide.md @@ -36,6 +36,9 @@ actions under `.github/`. and `make audit`, then delegates coverage generation to the shared coverage action. When the Rust extension is enabled, it also sets up Rust, installs Rust lint and test tools, and passes `rust_extension/Cargo.toml` to coverage. +- `.github/workflows/act-validation.yml` runs rendered workflow validation in a + separate workflow. It installs `act`, checks Docker availability, and runs + `make test WITH_ACT=1` outside the coverage path. - `.github/workflows/release.yml` publishes wheels when a `v*.*.*` tag is pushed. It builds a pure Python wheel, creates a GitHub release with generated release notes, downloads wheel artifacts, and uploads them to the tag release. diff --git a/tests/helpers/tooling_contracts.py b/tests/helpers/tooling_contracts.py index 3ee7cba..6d01fc2 100644 --- a/tests/helpers/tooling_contracts.py +++ b/tests/helpers/tooling_contracts.py @@ -63,6 +63,7 @@ def assert_generated_tooling_contracts( pyproject: dict[str, Any], makefile: str, ci_workflow: str, + act_validation_workflow: str, release_workflow: str, build_wheels_workflow: str, build_wheels_action: str, @@ -83,6 +84,8 @@ def assert_generated_tooling_contracts( UTF-8 text of the generated Makefile. ci_workflow : str UTF-8 text of the generated CI workflow. + act_validation_workflow : str + UTF-8 text of the generated act-validation workflow. release_workflow : str UTF-8 text of the generated release workflow. build_wheels_workflow : str @@ -115,6 +118,7 @@ def assert_generated_tooling_contracts( pyproject=pyproject, makefile=makefile, ci_workflow=ci_workflow, + act_validation_workflow=act_validation_workflow, release_workflow=release_workflow, build_wheels_workflow=build_wheels_workflow, build_wheels_action=build_wheels_action, @@ -141,6 +145,10 @@ def assert_generated_tooling_contracts( ci_workflow=ci_workflow, use_rust=use_rust, ) + _assert_act_validation_workflow_contracts( + act_validation_workflow=act_validation_workflow, + use_rust=use_rust, + ) _assert_release_workflow_contracts( release_workflow=release_workflow, use_rust=use_rust, @@ -568,6 +576,10 @@ def _assert_ci_workflow_contracts( assert "make audit" in ci_workflow, ( "expected generated CI workflow to run the dependency audit gate" ) + assert "make test WITH_ACT=1" not in ci_workflow, ( + "expected generated main CI workflow to leave act validation to a " + "separate workflow" + ) assert "make build" in ci_workflow, ( "expected generated CI workflow to build the project before checks" ) @@ -599,6 +611,35 @@ def _assert_ci_workflow_contracts( ) +def _assert_act_validation_workflow_contracts( + *, act_validation_workflow: str, use_rust: bool +) -> None: + """Assert generated act-validation workflow contracts.""" + assert "name: Act Validation" in act_validation_workflow, ( + "expected generated act-validation workflow to be named" + ) + assert "ACT_VERSION: v0.2.80" in act_validation_workflow, ( + "expected generated act-validation workflow to pin act" + ) + assert "act_Linux_x86_64.tar.gz" in act_validation_workflow, ( + "expected generated act-validation workflow to install act" + ) + assert "docker info" in act_validation_workflow, ( + "expected generated act-validation workflow to verify Docker" + ) + assert "make test WITH_ACT=1" in act_validation_workflow, ( + "expected generated act-validation workflow to run act-enabled tests" + ) + if use_rust: + assert "leynos/shared-actions/.github/actions/setup-rust" in ( + act_validation_workflow + ), "expected Rust variant act-validation workflow to set up Rust" + else: + assert "setup-rust" not in act_validation_workflow, ( + "expected pure-Python act-validation workflow to omit Rust setup" + ) + + def _assert_release_workflow_contracts( *, release_workflow: str, use_rust: bool ) -> None: diff --git a/tests/test_helpers.py b/tests/test_helpers.py index bb81004..f918c3f 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -350,12 +350,18 @@ def test_parent_makefile_test_target_uses_requisite_pytest_command() -> None: assert "uvx is required to run template tests" in makefile, ( "expected parent Makefile to fail early with a uvx installation message" ) + assert "WITH_ACT ?= 0" in makefile, ( + "expected parent Makefile to default act validation off" + ) + assert "RUN_ACT_VALIDATION=1" in makefile, ( + "expected parent Makefile to map WITH_ACT to act validation" + ) assert ( - "$(UV) --with pytest-copier --with pyyaml --with syrupy --with make-parser " - "pytest tests/" in makefile + "$(ACT_TEST_ENV) $(UV) --with pytest-copier --with pyyaml --with syrupy " + "--with make-parser pytest tests/" in makefile ), ( "expected parent Makefile test target to run pytest through $(UV) with " - "pytest-copier, pyyaml, syrupy, and make-parser" + "pytest-copier, pyyaml, syrupy, make-parser, and act environment wiring" ) diff --git a/tests/test_parent_ci.py b/tests/test_parent_ci.py new file mode 100644 index 0000000..0e3fbfb --- /dev/null +++ b/tests/test_parent_ci.py @@ -0,0 +1,49 @@ +"""Validate parent repository CI workflow contracts.""" + +from __future__ import annotations + +from pathlib import Path + +from tests.helpers.generated_files import read_generated_text + +REPO_ROOT = Path(__file__).resolve().parent.parent + + +def test_parent_ci_splits_application_and_act_validation_tests() -> None: + """Validate parent CI splits normal and act-enabled test gates. + + Parameters + ---------- + None + This test does not use pytest fixtures. + + Returns + ------- + None + The test passes when the parent CI workflow runs normal tests and the + separate act-validation workflow installs act prerequisites before + running ``make test WITH_ACT=1``. + """ + ci_workflow = read_generated_text(REPO_ROOT / ".github" / "workflows" / "ci.yml") + act_workflow = read_generated_text( + REPO_ROOT / ".github" / "workflows" / "act-validation.yml" + ) + + assert "make test\n" in ci_workflow, ( + "expected parent CI to run the normal parent test gate" + ) + assert "make test WITH_ACT=1" not in ci_workflow, ( + "expected parent CI to leave act validation to a separate workflow" + ) + assert "ACT_VERSION: v0.2.80" in act_workflow, ( + "expected parent act-validation workflow to pin the act release" + ) + assert "act_Linux_x86_64.tar.gz" in act_workflow, ( + "expected parent act-validation workflow to install the act Linux binary" + ) + assert "docker info" in act_workflow, ( + "expected parent act-validation workflow to verify Docker before act tests" + ) + assert "make test WITH_ACT=1" in act_workflow, ( + "expected parent act-validation workflow to run parent tests with act enabled" + ) diff --git a/tests/test_template.py b/tests/test_template.py index 756dfb1..a266f11 100644 --- a/tests/test_template.py +++ b/tests/test_template.py @@ -412,6 +412,9 @@ def test_generated_tooling_contracts( agents = read_generated_text(project / "AGENTS.md") makefile = read_generated_text(project / "Makefile") ci_workflow = read_generated_text(project / ".github" / "workflows" / "ci.yml") + act_validation_workflow = read_generated_text( + project / ".github" / "workflows" / "act-validation.yml" + ) release_workflow = read_generated_text( project / ".github" / "workflows" / "release.yml" ) @@ -431,6 +434,7 @@ def test_generated_tooling_contracts( pyproject=pyproject, makefile=makefile, ci_workflow=ci_workflow, + act_validation_workflow=act_validation_workflow, release_workflow=release_workflow, build_wheels_workflow=build_wheels_workflow, build_wheels_action=build_wheels_action, From a22c033fa934a21a30b70b8bae9b6f4131a735c2 Mon Sep 17 00:00:00 2001 From: Payton McIntosh Date: Thu, 4 Jun 2026 02:20:03 +0100 Subject: [PATCH 20/32] Stop installing mdformat-all in parent CI Remove the invalid uv tool installation for mdformat-all from both parent CI workflows. The parent workflows run tests, not make fmt, so they do not need the Markdown formatting wrapper or its mdtablefix dependency. Add workflow contract assertions so the bad install path is not reintroduced. --- .github/workflows/act-validation.yml | 1 - .github/workflows/ci.yml | 1 - tests/test_parent_ci.py | 12 ++++++++++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/act-validation.yml b/.github/workflows/act-validation.yml index cbeb338..df13d86 100644 --- a/.github/workflows/act-validation.yml +++ b/.github/workflows/act-validation.yml @@ -35,7 +35,6 @@ jobs: sudo tar -xz -C /usr/local/bin act npm install -g markdownlint-cli2 uv tool install mbake - uv tool install mdformat-all - name: Check Docker runtime run: docker info diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index df006f0..375d936 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,7 +31,6 @@ jobs: run: | npm install -g markdownlint-cli2 uv tool install mbake - uv tool install mdformat-all - name: Run template tests run: make test diff --git a/tests/test_parent_ci.py b/tests/test_parent_ci.py index 0e3fbfb..5cde6d2 100644 --- a/tests/test_parent_ci.py +++ b/tests/test_parent_ci.py @@ -32,6 +32,12 @@ def test_parent_ci_splits_application_and_act_validation_tests() -> None: assert "make test\n" in ci_workflow, ( "expected parent CI to run the normal parent test gate" ) + assert "uv tool install mdformat-all" not in ci_workflow, ( + "expected parent CI not to install mdformat-all through uv" + ) + assert "mdtablefix" not in ci_workflow, ( + "expected parent CI not to install Markdown formatting tools" + ) assert "make test WITH_ACT=1" not in ci_workflow, ( "expected parent CI to leave act validation to a separate workflow" ) @@ -41,6 +47,12 @@ def test_parent_ci_splits_application_and_act_validation_tests() -> None: assert "act_Linux_x86_64.tar.gz" in act_workflow, ( "expected parent act-validation workflow to install the act Linux binary" ) + assert "uv tool install mdformat-all" not in act_workflow, ( + "expected parent act-validation workflow not to install mdformat-all through uv" + ) + assert "mdtablefix" not in act_workflow, ( + "expected parent act-validation workflow not to install Markdown formatting tools" + ) assert "docker info" in act_workflow, ( "expected parent act-validation workflow to verify Docker before act tests" ) From c7e44d570292f415952b3a5b98a40a7be9e8068c Mon Sep 17 00:00:00 2001 From: Payton McIntosh Date: Thu, 4 Jun 2026 10:33:31 +0100 Subject: [PATCH 21/32] Always run Rust doctests in generated tests Run cargo test --doc for Rust-enabled generated projects regardless of whether cargo-nextest is available. Plain cargo test with --all-targets does not cover Rust doctests, so the fallback path previously let broken examples pass. Tighten the generated tooling contract to reject reintroducing the conditional nextest-only doctest path. --- template/Makefile.jinja | 2 -- tests/helpers/tooling_contracts.py | 7 +++++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/template/Makefile.jinja b/template/Makefile.jinja index 0da5f70..3882c4e 100644 --- a/template/Makefile.jinja +++ b/template/Makefile.jinja @@ -175,9 +175,7 @@ test: build $(VENV_TOOLS) ## Run tests exit 1; \ } RUSTFLAGS="$(RUST_FLAGS)" $(CARGO) $(TEST_CMD) $(TEST_FLAGS) $(BUILD_JOBS) -ifneq ($(TEST_CMD),test) RUSTFLAGS="$(RUST_FLAGS)" $(CARGO) test --doc --manifest-path $(RUST_CRATE_DIR)/Cargo.toml --all-features -endif {% endif %} help: ## Show available targets diff --git a/tests/helpers/tooling_contracts.py b/tests/helpers/tooling_contracts.py index 6d01fc2..a8c2f6a 100644 --- a/tests/helpers/tooling_contracts.py +++ b/tests/helpers/tooling_contracts.py @@ -525,6 +525,13 @@ def _assert_makefile_contracts(*, makefile: str, use_rust: bool) -> None: assert "$(CARGO) $(TEST_CMD) $(TEST_FLAGS)" in makefile, ( "expected Rust variant tests to use the selected cargo test command" ) + assert "ifneq ($(TEST_CMD),test)" not in makefile, ( + "expected Rust variant tests to run doctests even when nextest is absent" + ) + assert ( + 'RUSTFLAGS="$(RUST_FLAGS)" $(CARGO) test --doc ' + "--manifest-path $(RUST_CRATE_DIR)/Cargo.toml --all-features" + ) in makefile, "expected Rust variant tests to run Rust doctests" assert "rust-audit:" in makefile, ( "expected Rust variant to expose the rust-audit target" ) From c22ba2408290c7efe451ec38aee9b33121d6a43b Mon Sep 17 00:00:00 2001 From: Payton McIntosh Date: Thu, 4 Jun 2026 10:50:28 +0100 Subject: [PATCH 22/32] Harden parent workflow tool installs Restrict parent CI workflow token permissions, pin markdownlint-cli2 and mbake, and verify the downloaded act archive before installing it in act-validation. Split the large tooling contract helper into focused domain modules while keeping the public orchestration imports stable. Fix the Rust audit Makefile contract so the raw Makefile assertion checks $(CARGO) audit and the parsed recipe assertion matches make-parser output. --- .github/workflows/act-validation.yml | 20 +- .github/workflows/ci.yml | 11 +- tests/helpers/agents_contracts.py | 178 ++++++++ tests/helpers/ci_contracts.py | 210 +++++++++ tests/helpers/makefile_contracts.py | 142 ++++++ tests/helpers/pyproject_contracts.py | 61 +++ tests/helpers/release_contracts.py | 62 +++ tests/helpers/tooling_contracts.py | 661 ++------------------------- tests/helpers/wheel_contracts.py | 50 ++ tests/test_parent_ci.py | 33 +- 10 files changed, 786 insertions(+), 642 deletions(-) create mode 100644 tests/helpers/agents_contracts.py create mode 100644 tests/helpers/ci_contracts.py create mode 100644 tests/helpers/makefile_contracts.py create mode 100644 tests/helpers/pyproject_contracts.py create mode 100644 tests/helpers/release_contracts.py create mode 100644 tests/helpers/wheel_contracts.py diff --git a/.github/workflows/act-validation.yml b/.github/workflows/act-validation.yml index df13d86..da90edb 100644 --- a/.github/workflows/act-validation.yml +++ b/.github/workflows/act-validation.yml @@ -5,11 +5,16 @@ on: branches: [main] pull_request: +permissions: + contents: read + jobs: act-validation: runs-on: ubuntu-latest env: ACT_VERSION: v0.2.80 + MARKDOWNLINT_CLI2_VERSION: 0.22.1 + MBAKE_VERSION: 1.4.6 steps: - name: Check out repository uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 @@ -31,10 +36,17 @@ jobs: - name: Install template test tools run: | - curl -fsSL "https://github.com/nektos/act/releases/download/${ACT_VERSION}/act_Linux_x86_64.tar.gz" | - sudo tar -xz -C /usr/local/bin act - npm install -g markdownlint-cli2 - uv tool install mbake + act_archive=act_Linux_x86_64.tar.gz + act_url="https://github.com/nektos/act/releases/download/${ACT_VERSION}/${act_archive}" + curl -fsSLo "${act_archive}" "${act_url}" + act_sha256="$( + curl -fsSL "https://api.github.com/repos/nektos/act/releases/tags/${ACT_VERSION}" | + python -c 'import json, sys; release = json.load(sys.stdin); print(next(asset["digest"].split(":", 1)[1] for asset in release["assets"] if asset["name"] == "act_Linux_x86_64.tar.gz"))' + )" + printf '%s %s\n' "${act_sha256}" "${act_archive}" | sha256sum -c - + sudo tar -xzf "${act_archive}" -C /usr/local/bin act + npm install -g "markdownlint-cli2@${MARKDOWNLINT_CLI2_VERSION}" + uv tool install "mbake==${MBAKE_VERSION}" - name: Check Docker runtime run: docker info diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 375d936..8f4f801 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,13 @@ on: branches: [main] pull_request: +permissions: + contents: read + +env: + MARKDOWNLINT_CLI2_VERSION: 0.22.1 + MBAKE_VERSION: 1.4.6 + jobs: test: runs-on: ubuntu-latest @@ -29,8 +36,8 @@ jobs: - name: Install template test tools run: | - npm install -g markdownlint-cli2 - uv tool install mbake + npm install -g "markdownlint-cli2@${MARKDOWNLINT_CLI2_VERSION}" + uv tool install "mbake==${MBAKE_VERSION}" - name: Run template tests run: make test diff --git a/tests/helpers/agents_contracts.py b/tests/helpers/agents_contracts.py new file mode 100644 index 0000000..3ce35c4 --- /dev/null +++ b/tests/helpers/agents_contracts.py @@ -0,0 +1,178 @@ +"""Validate rendered AGENTS.md contracts against generated Makefiles. + +This module contains assistant-guidance assertions for rendered projects. It +uses ``tests.helpers.makefile_contracts`` to parse generated Makefiles so the +documented targets and command flags are checked against the actual recipes. +""" + +from __future__ import annotations + +import re + +from tests.helpers.makefile_contracts import _parse_makefile_rules + + +def _assert_agents_contracts(agents: str) -> None: + """Assert generated assistant guidance documents act-enabled testing.""" + assert "make test WITH_ACT=1" in agents, ( + "expected generated AGENTS.md to document act-enabled test runs" + ) + assert "RUN_ACT_VALIDATION=1" in agents, ( + "expected generated AGENTS.md to describe the pytest act environment" + ) + assert "meaningful, reviewer-useful contracts" in agents, ( + "expected generated AGENTS.md to require meaningful snapshot contracts" + ) + assert "generic dumps" in agents, ( + "expected generated AGENTS.md to warn against generic snapshots" + ) + assert "semantic assertions" in agents, ( + "expected generated AGENTS.md to pair snapshots with semantic assertions" + ) + assert "normalize nondeterministic fields" in agents, ( + "expected generated AGENTS.md to require nondeterministic field redaction" + ) + assert "brittle snapshots" in agents, ( + "expected generated AGENTS.md to warn against brittle snapshot churn" + ) + + +def _assert_agents_make_targets_mirror_makefile( + *, agents: str, makefile: str, package_name: str, use_rust: bool +) -> None: + """Assert AGENTS.md make target references match parsed Makefile commands.""" + documented_targets = _documented_make_targets(agents) + makefile_rules = _parse_makefile_rules(makefile) + makefile_targets = set(makefile_rules) + missing_targets = sorted(documented_targets - makefile_targets) + assert not missing_targets, ( + "expected every make target documented in generated AGENTS.md to exist " + f"in the generated Makefile, missing: {missing_targets}" + ) + required_documented_targets = { + "audit", + "check-fmt", + "fmt", + "lint", + "markdownlint", + "nixie", + "test", + "typecheck", + } + missing_documented_targets = sorted( + required_documented_targets - documented_targets + ) + assert not missing_documented_targets, ( + "expected generated AGENTS.md to document the generated Makefile quality " + f"gate targets, missing: {missing_documented_targets}" + ) + _assert_documented_command_flags( + agents=agents, + makefile_rules=makefile_rules, + package_name=package_name, + use_rust=use_rust, + ) + + +def _assert_documented_command_flags( + *, + agents: str, + makefile_rules: dict[str, list[str]], + package_name: str, + use_rust: bool, +) -> None: + """Assert documented make command flags are present in parsed recipes.""" + python_targets = f"{package_name} tests" + command_contracts = { + "check-fmt": [ + ("AGENTS.md", "ruff format --check $(PYTHON_TARGETS)"), + ("Makefile", f"ruff format --check {python_targets}"), + ], + "lint-python": [ + ("AGENTS.md", "ruff check $(PYTHON_TARGETS)"), + ("Makefile", f"ruff check {python_targets}"), + ], + "typecheck": [ + ("AGENTS.md", "ty check $(PYTHON_TARGETS)"), + ("Makefile", f"ty check {python_targets}"), + ], + "audit": [ + ("AGENTS.md", "pip-audit"), + ("Makefile", "pip-audit"), + ], + "test": [ + ("AGENTS.md", "pytest -v -n $(PYTEST_XDIST_WORKERS)"), + ("Makefile", "pytest -v -n auto"), + ], + } + if use_rust: + command_contracts.update( + { + "check-fmt": [ + *command_contracts["check-fmt"], + ( + "AGENTS.md", + "cargo fmt --manifest-path rust_extension/Cargo.toml", + ), + ("AGENTS.md", "--all -- --check"), + ("Makefile", "fmt --manifest-path rust_extension/Cargo.toml"), + ("Makefile", "--all -- --check"), + ], + "lint-rust": [ + ("AGENTS.md", "cargo doc --no-deps"), + ( + "AGENTS.md", + "cargo clippy --manifest-path rust_extension/Cargo.toml", + ), + ("AGENTS.md", "--all-targets --all-features -- -D warnings"), + ("AGENTS.md", "whitaker --all -- --all-targets --all-features"), + ("Makefile", "doc --no-deps"), + ("Makefile", "clippy --manifest-path rust_extension/Cargo.toml"), + ("Makefile", "--all-targets --all-features -- -D warnings"), + ("Makefile", "--all -- --all-targets --all-features"), + ], + "test": [ + *command_contracts["test"], + ( + "AGENTS.md", + "cargo nextest run --manifest-path rust_extension/Cargo.toml", + ), + ( + "AGENTS.md", + "cargo test --manifest-path rust_extension/Cargo.toml", + ), + ( + "AGENTS.md", + "cargo test --doc --manifest-path rust_extension/Cargo.toml", + ), + ("Makefile", "--manifest-path rust_extension/Cargo.toml"), + ("Makefile", "--all-targets --all-features"), + ( + "Makefile", + "test --doc --manifest-path rust_extension/Cargo.toml", + ), + ], + "audit": [ + *command_contracts["audit"], + ("AGENTS.md", "cargo audit"), + ("Makefile", "$(MAKE) rust-audit"), + ], + "rust-audit": [ + ("Makefile", "cd rust_extension"), + ("Makefile", ") audit"), + ], + } + ) + for target, checks in command_contracts.items(): + commands = "\n".join(makefile_rules[target]) + for source, fragment in checks: + haystack = agents if source == "AGENTS.md" else commands + assert fragment in haystack, ( + f"expected generated {source} {target!r} command contract to " + f"include {fragment!r}" + ) + + +def _documented_make_targets(agents: str) -> set[str]: + """Return make targets referenced in rendered AGENTS.md guidance.""" + return set(re.findall(r"`make ([a-zA-Z][a-zA-Z_-]*)", agents)) diff --git a/tests/helpers/ci_contracts.py b/tests/helpers/ci_contracts.py new file mode 100644 index 0000000..78f71d6 --- /dev/null +++ b/tests/helpers/ci_contracts.py @@ -0,0 +1,210 @@ +"""Validate rendered CI and act-validation workflow contracts. + +This module owns generated CI workflow assertions, including coverage-action +inputs, act-validation workflow structure, and shared checkout credential +checks. Release workflow helpers import ``_checkout_steps_disable_credentials`` +from here so checkout security behaviour is asserted consistently. +""" + +from __future__ import annotations + +from typing import Any + +from tests.helpers.generated_files import ( + parse_yaml_mapping, + require_mapping, + require_sequence, +) + + +def assert_ci_coverage_action_contract( + *, ci_workflow: str, package_name: str, use_rust: bool +) -> None: + """Assert generated CI coverage inputs used by act validation. + + Parameters + ---------- + ci_workflow : str + UTF-8 text of the generated CI workflow. + package_name : str + Generated Python import package name used to derive the coverage + artefact suffix. + use_rust : bool + Whether the rendered variant includes the optional Rust extension. + + Returns + ------- + None + The helper returns after the coverage action contract matches + expectations. + + Raises + ------ + AssertionError + Raised when checkout credentials, coverage action pinning, coverage + output settings, artefact naming, or Rust manifest inputs are wrong. + + Examples + -------- + Validate a rendered CI workflow's coverage step:: + + assert_ci_coverage_action_contract( + ci_workflow=ci_workflow, + package_name="example_pkg", + use_rust=True, + ) + """ + parsed_ci_workflow = _parse_ci_workflow(ci_workflow) + jobs = require_mapping(parsed_ci_workflow, "jobs", "CI workflow") + lint_test = require_mapping(jobs, "lint-test", "CI workflow jobs") + steps = require_sequence(lint_test, "steps", "CI lint-test job") + assert _checkout_steps_disable_credentials(steps), ( + "expected CI checkout steps to disable credential persistence" + ) + coverage_steps = [ + step + for step in steps + if isinstance(step, dict) and step.get("name") == "Test and Measure Coverage" + ] + assert len(coverage_steps) == 1, "expected one shared coverage action step" + coverage_step = coverage_steps[0] + assert ( + coverage_step.get("uses") + == "leynos/shared-actions/.github/actions/generate-coverage" + "@d400b079fb6a8fa92f7e7b6c57f3d1c92a4b2d54" + ), "expected CI to use the pinned shared coverage action" + coverage_inputs = require_mapping(coverage_step, "with", "coverage step") + assert coverage_inputs.get("output-path") == "coverage.xml", ( + "expected CI coverage output path to match the act assertion" + ) + assert coverage_inputs.get("format") == "cobertura", ( + "expected CI coverage format to match the CodeScene upload" + ) + assert coverage_inputs.get("artefact-name-suffix") == package_name.replace( + "_", "-" + ), "expected package-specific coverage artefact name suffix" + if use_rust: + assert coverage_inputs.get("cargo-manifest") == "rust_extension/Cargo.toml", ( + "expected Rust variant to pass the extension manifest to coverage" + ) + else: + assert "cargo-manifest" not in coverage_inputs, ( + "expected pure-Python variant to omit Rust coverage inputs" + ) + + +def _parse_ci_workflow(ci_workflow: str) -> dict[str, Any]: + """Parse a generated CI workflow as a YAML mapping.""" + return parse_yaml_mapping(ci_workflow, "CI workflow") + + +def _assert_ci_workflow_contracts( + *, parsed_ci_workflow: dict[str, Any], ci_workflow: str, use_rust: bool +) -> None: + """Assert generated CI workflow contracts. + + Parameters + ---------- + parsed_ci_workflow : dict[str, Any] + Parsed generated CI workflow mapping. + ci_workflow : str + UTF-8 text of the generated CI workflow, used for string-level contract + assertions. + use_rust : bool + Whether the rendered variant includes the optional Rust extension. + """ + jobs = require_mapping(parsed_ci_workflow, "jobs", "CI workflow") + lint_test = require_mapping(jobs, "lint-test", "CI workflow jobs") + steps = require_sequence(lint_test, "steps", "CI lint-test job") + assert _checkout_steps_disable_credentials(steps), ( + "expected generated CI workflow to disable checkout credentials" + ) + assert "make check-fmt" in ci_workflow, ( + "expected generated CI workflow to run the formatting gate" + ) + assert "make lint" in ci_workflow, ( + "expected generated CI workflow to run the lint gate" + ) + assert "make typecheck" in ci_workflow, ( + "expected generated CI workflow to run the typecheck gate" + ) + assert "make audit" in ci_workflow, ( + "expected generated CI workflow to run the dependency audit gate" + ) + assert "make test WITH_ACT=1" not in ci_workflow, ( + "expected generated main CI workflow to leave act validation to a " + "separate workflow" + ) + assert "make build" in ci_workflow, ( + "expected generated CI workflow to build the project before checks" + ) + assert "leynos/shared-actions/.github/actions/generate-coverage" in ci_workflow, ( + "expected generated CI workflow to use the shared coverage action" + ) + if use_rust: + assert "leynos/shared-actions/.github/actions/setup-rust" in ci_workflow, ( + "expected Rust variant CI to set up Rust" + ) + assert "Cache Rust lint and test tools" in ci_workflow, ( + "expected Rust variant CI to cache Rust tools" + ) + assert "cargo-manifest: rust_extension/Cargo.toml" in ci_workflow, ( + "expected Rust variant CI to pass the Rust manifest to coverage" + ) + assert "cargo install --locked cargo-audit" in ci_workflow, ( + "expected Rust variant CI to install cargo-audit" + ) + else: + assert "setup-rust" not in ci_workflow, ( + "expected pure-Python CI to omit Rust setup" + ) + assert "cargo-manifest" not in ci_workflow, ( + "expected pure-Python CI to omit Rust coverage inputs" + ) + assert "cargo-audit" not in ci_workflow, ( + "expected pure-Python CI to omit Rust audit installation" + ) + + +def _assert_act_validation_workflow_contracts( + *, act_validation_workflow: str, use_rust: bool +) -> None: + """Assert generated act-validation workflow contracts.""" + assert "name: Act Validation" in act_validation_workflow, ( + "expected generated act-validation workflow to be named" + ) + assert "ACT_VERSION: v0.2.80" in act_validation_workflow, ( + "expected generated act-validation workflow to pin act" + ) + assert "act_Linux_x86_64.tar.gz" in act_validation_workflow, ( + "expected generated act-validation workflow to install act" + ) + assert "docker info" in act_validation_workflow, ( + "expected generated act-validation workflow to verify Docker" + ) + assert "make test WITH_ACT=1" in act_validation_workflow, ( + "expected generated act-validation workflow to run act-enabled tests" + ) + if use_rust: + assert "leynos/shared-actions/.github/actions/setup-rust" in ( + act_validation_workflow + ), "expected Rust variant act-validation workflow to set up Rust" + else: + assert "setup-rust" not in act_validation_workflow, ( + "expected pure-Python act-validation workflow to omit Rust setup" + ) + + +def _checkout_steps_disable_credentials(steps: list[Any]) -> bool: + """Return whether all checkout steps disable credential persistence.""" + checkout_steps = [ + step + for step in steps + if isinstance(step, dict) + and str(step.get("uses", "")).startswith("actions/checkout@") + ] + return bool(checkout_steps) and all( + isinstance(step.get("with"), dict) + and step["with"].get("persist-credentials") is False + for step in checkout_steps + ) diff --git a/tests/helpers/makefile_contracts.py b/tests/helpers/makefile_contracts.py new file mode 100644 index 0000000..9e579c9 --- /dev/null +++ b/tests/helpers/makefile_contracts.py @@ -0,0 +1,142 @@ +"""Validate rendered Makefile contracts and parsed command recipes. + +This module owns Makefile-specific assertions for generated projects. The +AGENTS contract helper imports ``_parse_makefile_rules`` from here so +documented commands can be compared against the parsed recipes without keeping +all contract domains in one large module. +""" + +from __future__ import annotations + +import re +import tempfile +from pathlib import Path + +import make_parser + + +def assert_common_make_targets(makefile: str) -> None: + """Assert Makefile targets shared by all generated variants. + + Parameters + ---------- + makefile : str + UTF-8 text of a generated Makefile. + + Returns + ------- + None + The helper returns after all shared Makefile assertions pass. + + Raises + ------ + AssertionError + Raised when a shared generated Makefile target or cleanup path is + missing. + + Examples + -------- + Validate shared targets after reading a generated Makefile:: + + assert_common_make_targets(makefile) + """ + assert "lint-python: build" in makefile, "Makefile should expose lint-python" + assert "lint: lint-python" in makefile, "lint should delegate to lint-python" + assert "audit: build" in makefile, "Makefile should expose audit" + assert ".uv-cache .uv-tools" in makefile, "clean should remove uv state dirs" + + +def _parse_makefile_rules(makefile: str) -> dict[str, list[str]]: + """Return generated Makefile rules parsed through make-parser.""" + target_names = set( + re.findall(r"^([a-zA-Z][a-zA-Z_-]*):", makefile, flags=re.MULTILINE) + ) + normalised_targets = { + target: target.replace("-", "_") for target in target_names if "-" in target + } + + def normalise_target(match: re.Match[str]) -> str: + target = match.group(1) + return normalised_targets.get(target, target) + ":" + + normalised_makefile = re.sub( + r"^([a-zA-Z][a-zA-Z_-]*):", + normalise_target, + makefile.replace("?=", "="), + flags=re.MULTILINE, + ) + with tempfile.TemporaryDirectory() as tmp_dir: + makefile_path = Path(tmp_dir) / "Makefile" + makefile_path.write_text(normalised_makefile, encoding="utf-8") + parsed = make_parser.make_load(makefile_path) + normalised_rules = parsed["rules"] + return { + target: normalised_rules[normalised_targets.get(target, target)]["commands"] + for target in target_names + if normalised_targets.get(target, target) in normalised_rules + } + + +def _assert_makefile_contracts(*, makefile: str, use_rust: bool) -> None: + """Assert generated Makefile contracts for both template variants.""" + assert_common_make_targets(makefile) + assert "WITH_ACT ?= 0" in makefile, ( + "expected generated Makefile to default act validation off" + ) + assert "ACT_TEST_ENV =" in makefile, ( + "expected generated Makefile to map WITH_ACT to pytest environment" + ) + assert "RUN_ACT_VALIDATION=1" in makefile, ( + "expected generated Makefile to enable act validation for pytest" + ) + assert "$(UV_ENV) $(ACT_TEST_ENV) $(UV) run pytest" in makefile, ( + "expected generated test target to include the act test environment" + ) + assert "PYTHON_TARGETS ?=" in makefile, ( + "expected generated Makefile to define Python target selection" + ) + assert "PYLINT_PYPY_SHIM_REF ?=" in makefile, ( + "expected generated Makefile to expose the Pylint shim revision" + ) + assert "test: build $(VENV_TOOLS)" in makefile, ( + "expected generated Makefile test target to depend on the project env" + ) + assert "$(UV_ENV) $(UV) run pip-audit" in makefile, ( + "expected generated audit target to run pip-audit" + ) + if use_rust: + assert "TEST_CMD :=" in makefile, ( + "expected Rust variant to select nextest or cargo test" + ) + assert "lint-rust: build whitaker" in makefile, ( + "expected Rust variant to expose the Rust lint target" + ) + assert "cargo is required for Rust tests" in makefile, ( + "expected Rust variant to fail clearly without cargo" + ) + assert "$(CARGO) $(TEST_CMD) $(TEST_FLAGS)" in makefile, ( + "expected Rust variant tests to use the selected cargo test command" + ) + assert "ifneq ($(TEST_CMD),test)" not in makefile, ( + "expected Rust variant tests to run doctests even when nextest is absent" + ) + assert ( + 'RUSTFLAGS="$(RUST_FLAGS)" $(CARGO) test --doc ' + "--manifest-path $(RUST_CRATE_DIR)/Cargo.toml --all-features" + ) in makefile, "expected Rust variant tests to run Rust doctests" + assert "rust-audit:" in makefile, ( + "expected Rust variant to expose the rust-audit target" + ) + assert "cd $(RUST_CRATE_DIR) && $(CARGO) audit" in makefile, ( + "expected Rust variant audit target to run cargo audit" + ) + else: + assert "lint-rust" not in makefile, ( + "expected pure-Python variant to omit Rust lint targets" + ) + assert "TEST_CMD :=" not in makefile, ( + "expected pure-Python variant to omit Rust test command selection" + ) + assert "rust-audit" not in makefile, ( + "expected pure-Python variant to omit Rust audit targets" + ) diff --git a/tests/helpers/pyproject_contracts.py b/tests/helpers/pyproject_contracts.py new file mode 100644 index 0000000..8ed8408 --- /dev/null +++ b/tests/helpers/pyproject_contracts.py @@ -0,0 +1,61 @@ +"""Validate rendered pyproject packaging contracts. + +This module contains pyproject-specific assertions extracted from +``tests.helpers.tooling_contracts``. The top-level tooling-contract +orchestrator imports this private helper so packaging checks remain isolated +from Makefile and workflow assertions while sharing the generated-file parsing +helpers. +""" + +from __future__ import annotations + +from typing import Any + +from tests.helpers.generated_files import require_mapping + + +def _assert_pyproject_contracts( + *, package_name: str, pyproject: dict[str, Any], use_rust: bool +) -> None: + """Assert generated Python packaging contracts.""" + project = require_mapping(pyproject, "project", "pyproject.toml") + assert project.get("name"), "expected generated project metadata to include a name" + assert project.get("requires-python") == ">=3.10", ( + "expected generated pyproject.toml to use the requested Python version" + ) + dependency_groups = require_mapping( + pyproject, + "dependency-groups", + "pyproject.toml", + ) + dev_dependencies = dependency_groups.get("dev") + assert isinstance(dev_dependencies, list), ( + "expected generated pyproject.toml to include a dev dependency group" + ) + for dependency in ["pytest", "pip-audit", "ruff", "pyright", "ty", "pytest-xdist"]: + assert dependency in dev_dependencies, ( + f"expected generated dev dependencies to include {dependency}" + ) + + build_system = require_mapping(pyproject, "build-system", "pyproject.toml") + if use_rust: + assert build_system.get("build-backend") == "maturin", ( + "expected Rust variant to use maturin as the build backend" + ) + maturin = require_mapping(pyproject, "tool", "pyproject.toml").get("maturin") + assert isinstance(maturin, dict), ( + "expected Rust variant to include tool.maturin configuration" + ) + assert maturin.get("manifest-path") == "rust_extension/Cargo.toml", ( + "expected Rust variant to point maturin at the extension manifest" + ) + assert maturin.get("module-name") == f"{package_name}._{package_name}_rs", ( + "expected Rust variant to render the package-specific module name" + ) + else: + assert build_system.get("build-backend") == "hatchling.build", ( + "expected pure-Python variant to use hatchling as the build backend" + ) + assert "maturin" not in pyproject.get("tool", {}), ( + "expected pure-Python variant to omit maturin configuration" + ) diff --git a/tests/helpers/release_contracts.py b/tests/helpers/release_contracts.py new file mode 100644 index 0000000..9bb0842 --- /dev/null +++ b/tests/helpers/release_contracts.py @@ -0,0 +1,62 @@ +"""Validate rendered release workflow contracts. + +Release workflow assertions live here to keep release-specific job wiring and +asset-upload checks separate from CI, Makefile, and wheel-action contracts. +The helper imports the shared checkout credential assertion from +``tests.helpers.ci_contracts``. +""" + +from __future__ import annotations + +from tests.helpers.ci_contracts import _checkout_steps_disable_credentials +from tests.helpers.generated_files import ( + parse_yaml_mapping, + require_mapping, + require_sequence, +) + + +def _assert_release_workflow_contracts( + *, release_workflow: str, use_rust: bool +) -> None: + """Assert generated release workflow contracts.""" + parsed_release_workflow = parse_yaml_mapping(release_workflow, "release workflow") + jobs = require_mapping(parsed_release_workflow, "jobs", "release workflow") + release = require_mapping(jobs, "release", "release workflow jobs") + release_steps = require_sequence(release, "steps", "release job") + assert _checkout_steps_disable_credentials(release_steps), ( + "expected generated release workflow to disable checkout credentials" + ) + assert "softprops/action-gh-release@v2" in release_workflow, ( + "expected release workflow to create a GitHub release" + ) + assert "actions/download-artifact@v4" in release_workflow, ( + "expected release workflow to download wheel artefacts" + ) + assert "--clobber" in release_workflow, ( + "expected release workflow to overwrite existing wheel assets on rerun" + ) + if use_rust: + assert "build-wheels" in jobs, ( + "expected Rust variant release workflow to build platform wheels" + ) + build_wheels = require_mapping(jobs, "build-wheels", "release workflow jobs") + assert build_wheels.get("uses") == "./.github/workflows/build-wheels.yml", ( + "expected Rust variant release workflow to call build-wheels.yml" + ) + assert release.get("needs") == ["build-wheels"], ( + "expected Rust variant release job to wait for platform wheels" + ) + assert "pure-wheel" not in jobs, ( + "expected Rust variant release workflow to skip pure wheel job" + ) + else: + assert "pure-wheel" in jobs, ( + "expected pure-Python release workflow to build one pure wheel" + ) + assert release.get("needs") == ["pure-wheel"], ( + "expected pure-Python release job to wait for the pure wheel" + ) + assert "build-wheels" not in jobs, ( + "expected pure-Python release workflow to skip platform wheel matrix" + ) diff --git a/tests/helpers/tooling_contracts.py b/tests/helpers/tooling_contracts.py index a8c2f6a..19569a9 100644 --- a/tests/helpers/tooling_contracts.py +++ b/tests/helpers/tooling_contracts.py @@ -1,59 +1,40 @@ -"""Assert rendered tooling contracts for generated project variants. - -This module contains the higher-level contract checks used after -``tests.helpers.rendering`` renders a project and -``tests.helpers.generated_files`` parses its structured files. The public -helpers validate generated Makefile targets, packaging metadata, assistant -guidance, and GitHub workflow structure for both pure-Python and Python/Rust -variants. Keep workflow and tooling assertions here so top-level template -tests stay focused on scenario setup. +"""Orchestrate rendered tooling contracts for generated project variants. + +This module keeps the public test-helper API stable while delegating +domain-specific assertions to focused helper modules: +``pyproject_contracts``, ``agents_contracts``, ``makefile_contracts``, +``ci_contracts``, ``release_contracts``, and ``wheel_contracts``. Top-level +template tests import from here, while each delegated module stays small enough +to own one contract domain. """ from __future__ import annotations -import re -import tempfile -from pathlib import Path from typing import Any -import make_parser - -from tests.helpers.generated_files import ( - parse_yaml_mapping, - require_mapping, - require_sequence, +from tests.helpers.agents_contracts import ( + _assert_agents_contracts, + _assert_agents_make_targets_mirror_makefile, ) +from tests.helpers.ci_contracts import ( + _assert_act_validation_workflow_contracts, + _assert_ci_workflow_contracts, + _parse_ci_workflow, + assert_ci_coverage_action_contract, +) +from tests.helpers.makefile_contracts import ( + _assert_makefile_contracts, + assert_common_make_targets, +) +from tests.helpers.pyproject_contracts import _assert_pyproject_contracts +from tests.helpers.release_contracts import _assert_release_workflow_contracts +from tests.helpers.wheel_contracts import _assert_wheel_workflow_contracts - -def assert_common_make_targets(makefile: str) -> None: - """Assert Makefile targets shared by all generated variants. - - Parameters - ---------- - makefile : str - UTF-8 text of a generated Makefile. - - Returns - ------- - None - The helper returns after all shared Makefile assertions pass. - - Raises - ------ - AssertionError - Raised when a shared generated Makefile target or cleanup path is - missing. - - Examples - -------- - Validate shared targets after reading a generated Makefile:: - - assert_common_make_targets(makefile) - """ - assert "lint-python: build" in makefile, "Makefile should expose lint-python" - assert "lint: lint-python" in makefile, "lint should delegate to lint-python" - assert "audit: build" in makefile, "Makefile should expose audit" - assert ".uv-cache .uv-tools" in makefile, "clean should remove uv state dirs" +__all__ = [ + "assert_ci_coverage_action_contract", + "assert_common_make_targets", + "assert_generated_tooling_contracts", +] def assert_generated_tooling_contracts( @@ -158,587 +139,3 @@ def assert_generated_tooling_contracts( build_wheels_action=build_wheels_action, pure_wheel_action=pure_wheel_action, ) - - -def assert_ci_coverage_action_contract( - *, ci_workflow: str, package_name: str, use_rust: bool -) -> None: - """Assert generated CI coverage inputs used by act validation. - - Parameters - ---------- - ci_workflow : str - UTF-8 text of the generated CI workflow. - package_name : str - Generated Python import package name used to derive the coverage - artefact suffix. - use_rust : bool - Whether the rendered variant includes the optional Rust extension. - - Returns - ------- - None - The helper returns after the coverage action contract matches - expectations. - - Raises - ------ - AssertionError - Raised when checkout credentials, coverage action pinning, coverage - output settings, artefact naming, or Rust manifest inputs are wrong. - - Examples - -------- - Validate a rendered CI workflow's coverage step:: - - assert_ci_coverage_action_contract( - ci_workflow=ci_workflow, - package_name="example_pkg", - use_rust=True, - ) - """ - parsed_ci_workflow = _parse_ci_workflow(ci_workflow) - jobs = require_mapping(parsed_ci_workflow, "jobs", "CI workflow") - lint_test = require_mapping(jobs, "lint-test", "CI workflow jobs") - steps = require_sequence(lint_test, "steps", "CI lint-test job") - assert _checkout_steps_disable_credentials(steps), ( - "expected CI checkout steps to disable credential persistence" - ) - coverage_steps = [ - step - for step in steps - if isinstance(step, dict) and step.get("name") == "Test and Measure Coverage" - ] - assert len(coverage_steps) == 1, "expected one shared coverage action step" - coverage_step = coverage_steps[0] - assert ( - coverage_step.get("uses") - == "leynos/shared-actions/.github/actions/generate-coverage" - "@d400b079fb6a8fa92f7e7b6c57f3d1c92a4b2d54" - ), "expected CI to use the pinned shared coverage action" - coverage_inputs = require_mapping(coverage_step, "with", "coverage step") - assert coverage_inputs.get("output-path") == "coverage.xml", ( - "expected CI coverage output path to match the act assertion" - ) - assert coverage_inputs.get("format") == "cobertura", ( - "expected CI coverage format to match the CodeScene upload" - ) - assert coverage_inputs.get("artefact-name-suffix") == package_name.replace( - "_", "-" - ), "expected package-specific coverage artefact name suffix" - if use_rust: - assert coverage_inputs.get("cargo-manifest") == "rust_extension/Cargo.toml", ( - "expected Rust variant to pass the extension manifest to coverage" - ) - else: - assert "cargo-manifest" not in coverage_inputs, ( - "expected pure-Python variant to omit Rust coverage inputs" - ) - - -def _parse_ci_workflow(ci_workflow: str) -> dict[str, Any]: - """Parse a generated CI workflow as a YAML mapping.""" - return parse_yaml_mapping(ci_workflow, "CI workflow") - - -def _assert_pyproject_contracts( - *, package_name: str, pyproject: dict[str, Any], use_rust: bool -) -> None: - """Assert generated Python packaging contracts.""" - project = require_mapping(pyproject, "project", "pyproject.toml") - assert project.get("name"), "expected generated project metadata to include a name" - assert project.get("requires-python") == ">=3.10", ( - "expected generated pyproject.toml to use the requested Python version" - ) - dependency_groups = require_mapping( - pyproject, - "dependency-groups", - "pyproject.toml", - ) - dev_dependencies = dependency_groups.get("dev") - assert isinstance(dev_dependencies, list), ( - "expected generated pyproject.toml to include a dev dependency group" - ) - for dependency in ["pytest", "pip-audit", "ruff", "pyright", "ty", "pytest-xdist"]: - assert dependency in dev_dependencies, ( - f"expected generated dev dependencies to include {dependency}" - ) - - build_system = require_mapping(pyproject, "build-system", "pyproject.toml") - if use_rust: - assert build_system.get("build-backend") == "maturin", ( - "expected Rust variant to use maturin as the build backend" - ) - maturin = require_mapping(pyproject, "tool", "pyproject.toml").get("maturin") - assert isinstance(maturin, dict), ( - "expected Rust variant to include tool.maturin configuration" - ) - assert maturin.get("manifest-path") == "rust_extension/Cargo.toml", ( - "expected Rust variant to point maturin at the extension manifest" - ) - assert maturin.get("module-name") == f"{package_name}._{package_name}_rs", ( - "expected Rust variant to render the package-specific module name" - ) - else: - assert build_system.get("build-backend") == "hatchling.build", ( - "expected pure-Python variant to use hatchling as the build backend" - ) - assert "maturin" not in pyproject.get("tool", {}), ( - "expected pure-Python variant to omit maturin configuration" - ) - - -def _assert_agents_contracts(agents: str) -> None: - """Assert generated assistant guidance documents act-enabled testing.""" - assert "make test WITH_ACT=1" in agents, ( - "expected generated AGENTS.md to document act-enabled test runs" - ) - assert "RUN_ACT_VALIDATION=1" in agents, ( - "expected generated AGENTS.md to describe the pytest act environment" - ) - assert "meaningful, reviewer-useful contracts" in agents, ( - "expected generated AGENTS.md to require meaningful snapshot contracts" - ) - assert "generic dumps" in agents, ( - "expected generated AGENTS.md to warn against generic snapshots" - ) - assert "semantic assertions" in agents, ( - "expected generated AGENTS.md to pair snapshots with semantic assertions" - ) - assert "normalize nondeterministic fields" in agents, ( - "expected generated AGENTS.md to require nondeterministic field redaction" - ) - assert "brittle snapshots" in agents, ( - "expected generated AGENTS.md to warn against brittle snapshot churn" - ) - - -def _assert_agents_make_targets_mirror_makefile( - *, agents: str, makefile: str, package_name: str, use_rust: bool -) -> None: - """Assert AGENTS.md make target references match parsed Makefile commands.""" - documented_targets = _documented_make_targets(agents) - makefile_rules = _parse_makefile_rules(makefile) - makefile_targets = set(makefile_rules) - missing_targets = sorted(documented_targets - makefile_targets) - assert not missing_targets, ( - "expected every make target documented in generated AGENTS.md to exist " - f"in the generated Makefile, missing: {missing_targets}" - ) - required_documented_targets = { - "audit", - "check-fmt", - "fmt", - "lint", - "markdownlint", - "nixie", - "test", - "typecheck", - } - missing_documented_targets = sorted( - required_documented_targets - documented_targets - ) - assert not missing_documented_targets, ( - "expected generated AGENTS.md to document the generated Makefile quality " - f"gate targets, missing: {missing_documented_targets}" - ) - _assert_documented_command_flags( - agents=agents, - makefile_rules=makefile_rules, - package_name=package_name, - use_rust=use_rust, - ) - - -def _assert_documented_command_flags( - *, - agents: str, - makefile_rules: dict[str, list[str]], - package_name: str, - use_rust: bool, -) -> None: - """Assert documented make command flags are present in parsed recipes.""" - python_targets = f"{package_name} tests" - command_contracts = { - "check-fmt": [ - ("AGENTS.md", "ruff format --check $(PYTHON_TARGETS)"), - ("Makefile", f"ruff format --check {python_targets}"), - ], - "lint-python": [ - ("AGENTS.md", "ruff check $(PYTHON_TARGETS)"), - ("Makefile", f"ruff check {python_targets}"), - ], - "typecheck": [ - ("AGENTS.md", "ty check $(PYTHON_TARGETS)"), - ("Makefile", f"ty check {python_targets}"), - ], - "audit": [ - ("AGENTS.md", "pip-audit"), - ("Makefile", "pip-audit"), - ], - "test": [ - ("AGENTS.md", "pytest -v -n $(PYTEST_XDIST_WORKERS)"), - ("Makefile", "pytest -v -n auto"), - ], - } - if use_rust: - command_contracts.update( - { - "check-fmt": [ - *command_contracts["check-fmt"], - ( - "AGENTS.md", - "cargo fmt --manifest-path rust_extension/Cargo.toml", - ), - ("AGENTS.md", "--all -- --check"), - ("Makefile", "fmt --manifest-path rust_extension/Cargo.toml"), - ("Makefile", "--all -- --check"), - ], - "lint-rust": [ - ("AGENTS.md", "cargo doc --no-deps"), - ( - "AGENTS.md", - "cargo clippy --manifest-path rust_extension/Cargo.toml", - ), - ("AGENTS.md", "--all-targets --all-features -- -D warnings"), - ("AGENTS.md", "whitaker --all -- --all-targets --all-features"), - ("Makefile", "doc --no-deps"), - ("Makefile", "clippy --manifest-path rust_extension/Cargo.toml"), - ("Makefile", "--all-targets --all-features -- -D warnings"), - ("Makefile", "--all -- --all-targets --all-features"), - ], - "test": [ - *command_contracts["test"], - ( - "AGENTS.md", - "cargo nextest run --manifest-path rust_extension/Cargo.toml", - ), - ( - "AGENTS.md", - "cargo test --manifest-path rust_extension/Cargo.toml", - ), - ( - "AGENTS.md", - "cargo test --doc --manifest-path rust_extension/Cargo.toml", - ), - ("Makefile", "--manifest-path rust_extension/Cargo.toml"), - ("Makefile", "--all-targets --all-features"), - ( - "Makefile", - "test --doc --manifest-path rust_extension/Cargo.toml", - ), - ], - "audit": [ - *command_contracts["audit"], - ("AGENTS.md", "cargo audit"), - ("Makefile", "$(MAKE) rust-audit"), - ], - "rust-audit": [ - ("Makefile", "cd rust_extension"), - ("Makefile", "cargo) audit"), - ], - } - ) - for target, checks in command_contracts.items(): - commands = "\n".join(makefile_rules[target]) - for source, fragment in checks: - haystack = agents if source == "AGENTS.md" else commands - assert fragment in haystack, ( - f"expected generated {source} {target!r} command contract to " - f"include {fragment!r}" - ) - - -def _documented_make_targets(agents: str) -> set[str]: - """Return make targets referenced in rendered AGENTS.md guidance.""" - return set(re.findall(r"`make ([a-zA-Z][a-zA-Z_-]*)", agents)) - - -def _parse_makefile_rules(makefile: str) -> dict[str, list[str]]: - """Return generated Makefile rules parsed through make-parser.""" - target_names = set( - re.findall(r"^([a-zA-Z][a-zA-Z_-]*):", makefile, flags=re.MULTILINE) - ) - normalised_targets = { - target: target.replace("-", "_") for target in target_names if "-" in target - } - - def normalise_target(match: re.Match[str]) -> str: - target = match.group(1) - return normalised_targets.get(target, target) + ":" - - normalised_makefile = re.sub( - r"^([a-zA-Z][a-zA-Z_-]*):", - normalise_target, - makefile.replace("?=", "="), - flags=re.MULTILINE, - ) - with tempfile.TemporaryDirectory() as tmp_dir: - makefile_path = Path(tmp_dir) / "Makefile" - makefile_path.write_text(normalised_makefile, encoding="utf-8") - parsed = make_parser.make_load(makefile_path) - normalised_rules = parsed["rules"] - return { - target: normalised_rules[normalised_targets.get(target, target)]["commands"] - for target in target_names - if normalised_targets.get(target, target) in normalised_rules - } - - -def _assert_makefile_contracts(*, makefile: str, use_rust: bool) -> None: - """Assert generated Makefile contracts for both template variants.""" - assert_common_make_targets(makefile) - assert "WITH_ACT ?= 0" in makefile, ( - "expected generated Makefile to default act validation off" - ) - assert "ACT_TEST_ENV =" in makefile, ( - "expected generated Makefile to map WITH_ACT to pytest environment" - ) - assert "RUN_ACT_VALIDATION=1" in makefile, ( - "expected generated Makefile to enable act validation for pytest" - ) - assert "$(UV_ENV) $(ACT_TEST_ENV) $(UV) run pytest" in makefile, ( - "expected generated test target to include the act test environment" - ) - assert "PYTHON_TARGETS ?=" in makefile, ( - "expected generated Makefile to define Python target selection" - ) - assert "PYLINT_PYPY_SHIM_REF ?=" in makefile, ( - "expected generated Makefile to expose the Pylint shim revision" - ) - assert "test: build $(VENV_TOOLS)" in makefile, ( - "expected generated Makefile test target to depend on the project env" - ) - assert "$(UV_ENV) $(UV) run pip-audit" in makefile, ( - "expected generated audit target to run pip-audit" - ) - if use_rust: - assert "TEST_CMD :=" in makefile, ( - "expected Rust variant to select nextest or cargo test" - ) - assert "lint-rust: build whitaker" in makefile, ( - "expected Rust variant to expose the Rust lint target" - ) - assert "cargo is required for Rust tests" in makefile, ( - "expected Rust variant to fail clearly without cargo" - ) - assert "$(CARGO) $(TEST_CMD) $(TEST_FLAGS)" in makefile, ( - "expected Rust variant tests to use the selected cargo test command" - ) - assert "ifneq ($(TEST_CMD),test)" not in makefile, ( - "expected Rust variant tests to run doctests even when nextest is absent" - ) - assert ( - 'RUSTFLAGS="$(RUST_FLAGS)" $(CARGO) test --doc ' - "--manifest-path $(RUST_CRATE_DIR)/Cargo.toml --all-features" - ) in makefile, "expected Rust variant tests to run Rust doctests" - assert "rust-audit:" in makefile, ( - "expected Rust variant to expose the rust-audit target" - ) - assert "cd $(RUST_CRATE_DIR) && $(CARGO) audit" in makefile, ( - "expected Rust variant audit target to run cargo audit" - ) - else: - assert "lint-rust" not in makefile, ( - "expected pure-Python variant to omit Rust lint targets" - ) - assert "TEST_CMD :=" not in makefile, ( - "expected pure-Python variant to omit Rust test command selection" - ) - assert "rust-audit" not in makefile, ( - "expected pure-Python variant to omit Rust audit targets" - ) - - -def _assert_ci_workflow_contracts( - *, parsed_ci_workflow: dict[str, Any], ci_workflow: str, use_rust: bool -) -> None: - """Assert generated CI workflow contracts. - - Parameters - ---------- - parsed_ci_workflow : dict[str, Any] - Parsed generated CI workflow mapping. - ci_workflow : str - UTF-8 text of the generated CI workflow, used for string-level contract - assertions. - use_rust : bool - Whether the rendered variant includes the optional Rust extension. - """ - jobs = require_mapping(parsed_ci_workflow, "jobs", "CI workflow") - lint_test = require_mapping(jobs, "lint-test", "CI workflow jobs") - steps = require_sequence(lint_test, "steps", "CI lint-test job") - assert _checkout_steps_disable_credentials(steps), ( - "expected generated CI workflow to disable checkout credentials" - ) - assert "make check-fmt" in ci_workflow, ( - "expected generated CI workflow to run the formatting gate" - ) - assert "make lint" in ci_workflow, ( - "expected generated CI workflow to run the lint gate" - ) - assert "make typecheck" in ci_workflow, ( - "expected generated CI workflow to run the typecheck gate" - ) - assert "make audit" in ci_workflow, ( - "expected generated CI workflow to run the dependency audit gate" - ) - assert "make test WITH_ACT=1" not in ci_workflow, ( - "expected generated main CI workflow to leave act validation to a " - "separate workflow" - ) - assert "make build" in ci_workflow, ( - "expected generated CI workflow to build the project before checks" - ) - assert "leynos/shared-actions/.github/actions/generate-coverage" in ci_workflow, ( - "expected generated CI workflow to use the shared coverage action" - ) - if use_rust: - assert "leynos/shared-actions/.github/actions/setup-rust" in ci_workflow, ( - "expected Rust variant CI to set up Rust" - ) - assert "Cache Rust lint and test tools" in ci_workflow, ( - "expected Rust variant CI to cache Rust tools" - ) - assert "cargo-manifest: rust_extension/Cargo.toml" in ci_workflow, ( - "expected Rust variant CI to pass the Rust manifest to coverage" - ) - assert "cargo install --locked cargo-audit" in ci_workflow, ( - "expected Rust variant CI to install cargo-audit" - ) - else: - assert "setup-rust" not in ci_workflow, ( - "expected pure-Python CI to omit Rust setup" - ) - assert "cargo-manifest" not in ci_workflow, ( - "expected pure-Python CI to omit Rust coverage inputs" - ) - assert "cargo-audit" not in ci_workflow, ( - "expected pure-Python CI to omit Rust audit installation" - ) - - -def _assert_act_validation_workflow_contracts( - *, act_validation_workflow: str, use_rust: bool -) -> None: - """Assert generated act-validation workflow contracts.""" - assert "name: Act Validation" in act_validation_workflow, ( - "expected generated act-validation workflow to be named" - ) - assert "ACT_VERSION: v0.2.80" in act_validation_workflow, ( - "expected generated act-validation workflow to pin act" - ) - assert "act_Linux_x86_64.tar.gz" in act_validation_workflow, ( - "expected generated act-validation workflow to install act" - ) - assert "docker info" in act_validation_workflow, ( - "expected generated act-validation workflow to verify Docker" - ) - assert "make test WITH_ACT=1" in act_validation_workflow, ( - "expected generated act-validation workflow to run act-enabled tests" - ) - if use_rust: - assert "leynos/shared-actions/.github/actions/setup-rust" in ( - act_validation_workflow - ), "expected Rust variant act-validation workflow to set up Rust" - else: - assert "setup-rust" not in act_validation_workflow, ( - "expected pure-Python act-validation workflow to omit Rust setup" - ) - - -def _assert_release_workflow_contracts( - *, release_workflow: str, use_rust: bool -) -> None: - """Assert generated release workflow contracts.""" - parsed_release_workflow = parse_yaml_mapping(release_workflow, "release workflow") - jobs = require_mapping(parsed_release_workflow, "jobs", "release workflow") - release = require_mapping(jobs, "release", "release workflow jobs") - release_steps = require_sequence(release, "steps", "release job") - assert _checkout_steps_disable_credentials(release_steps), ( - "expected generated release workflow to disable checkout credentials" - ) - assert "softprops/action-gh-release@v2" in release_workflow, ( - "expected release workflow to create a GitHub release" - ) - assert "actions/download-artifact@v4" in release_workflow, ( - "expected release workflow to download wheel artefacts" - ) - assert "--clobber" in release_workflow, ( - "expected release workflow to overwrite existing wheel assets on rerun" - ) - if use_rust: - assert "build-wheels" in jobs, ( - "expected Rust variant release workflow to build platform wheels" - ) - build_wheels = require_mapping(jobs, "build-wheels", "release workflow jobs") - assert build_wheels.get("uses") == "./.github/workflows/build-wheels.yml", ( - "expected Rust variant release workflow to call build-wheels.yml" - ) - assert release.get("needs") == ["build-wheels"], ( - "expected Rust variant release job to wait for platform wheels" - ) - assert "pure-wheel" not in jobs, ( - "expected Rust variant release workflow to skip pure wheel job" - ) - else: - assert "pure-wheel" in jobs, ( - "expected pure-Python release workflow to build one pure wheel" - ) - assert release.get("needs") == ["pure-wheel"], ( - "expected pure-Python release job to wait for the pure wheel" - ) - assert "build-wheels" not in jobs, ( - "expected pure-Python release workflow to skip platform wheel matrix" - ) - - -def _assert_wheel_workflow_contracts( - *, - build_wheels_workflow: str, - build_wheels_action: str, - pure_wheel_action: str, -) -> None: - """Assert generated wheel workflow and action contracts.""" - parsed_build_wheels = parse_yaml_mapping( - build_wheels_workflow, - "build-wheels workflow", - ) - build_jobs = require_mapping(parsed_build_wheels, "jobs", "build-wheels workflow") - build_job = require_mapping(build_jobs, "build", "build-wheels workflow jobs") - strategy = require_mapping(build_job, "strategy", "build-wheels build job") - matrix = require_mapping(strategy, "matrix", "build-wheels strategy") - includes = require_sequence(matrix, "include", "build-wheels matrix") - assert len(includes) == 6, ( - "expected build-wheels workflow to cover Linux, Windows, and macOS" - ) - assert "persist-credentials: false" in build_wheels_workflow, ( - "expected build-wheels workflow checkout to disable credentials" - ) - assert "uvx --with 'cibuildwheel>=2.16.0,<4.0.0' cibuildwheel" in ( - build_wheels_action - ), "expected Rust wheel action to build through cibuildwheel" - assert "persist-credentials: false" in build_wheels_action, ( - "expected build-wheels action checkout to disable credentials" - ) - assert "uv build --wheel" in pure_wheel_action, ( - "expected pure-wheel action to build through uv" - ) - assert "persist-credentials: false" in pure_wheel_action, ( - "expected pure-wheel action checkout to disable credentials" - ) - - -def _checkout_steps_disable_credentials(steps: list[Any]) -> bool: - """Return whether all checkout steps disable credential persistence.""" - checkout_steps = [ - step - for step in steps - if isinstance(step, dict) - and str(step.get("uses", "")).startswith("actions/checkout@") - ] - return bool(checkout_steps) and all( - isinstance(step.get("with"), dict) - and step["with"].get("persist-credentials") is False - for step in checkout_steps - ) diff --git a/tests/helpers/wheel_contracts.py b/tests/helpers/wheel_contracts.py new file mode 100644 index 0000000..1fbe895 --- /dev/null +++ b/tests/helpers/wheel_contracts.py @@ -0,0 +1,50 @@ +"""Validate rendered wheel workflow and composite action contracts. + +Wheel-building assertions are separated from release workflow checks so matrix +coverage, checkout hardening, and build-command contracts remain focused on the +workflow/action files that produce wheel artefacts. +""" + +from __future__ import annotations + +from tests.helpers.generated_files import ( + parse_yaml_mapping, + require_mapping, + require_sequence, +) + + +def _assert_wheel_workflow_contracts( + *, + build_wheels_workflow: str, + build_wheels_action: str, + pure_wheel_action: str, +) -> None: + """Assert generated wheel workflow and action contracts.""" + parsed_build_wheels = parse_yaml_mapping( + build_wheels_workflow, + "build-wheels workflow", + ) + build_jobs = require_mapping(parsed_build_wheels, "jobs", "build-wheels workflow") + build_job = require_mapping(build_jobs, "build", "build-wheels workflow jobs") + strategy = require_mapping(build_job, "strategy", "build-wheels build job") + matrix = require_mapping(strategy, "matrix", "build-wheels strategy") + includes = require_sequence(matrix, "include", "build-wheels matrix") + assert len(includes) == 6, ( + "expected build-wheels workflow to cover Linux, Windows, and macOS" + ) + assert "persist-credentials: false" in build_wheels_workflow, ( + "expected build-wheels workflow checkout to disable credentials" + ) + assert "uvx --with 'cibuildwheel>=2.16.0,<4.0.0' cibuildwheel" in ( + build_wheels_action + ), "expected Rust wheel action to build through cibuildwheel" + assert "persist-credentials: false" in build_wheels_action, ( + "expected build-wheels action checkout to disable credentials" + ) + assert "uv build --wheel" in pure_wheel_action, ( + "expected pure-wheel action to build through uv" + ) + assert "persist-credentials: false" in pure_wheel_action, ( + "expected pure-wheel action checkout to disable credentials" + ) diff --git a/tests/test_parent_ci.py b/tests/test_parent_ci.py index 5cde6d2..e26b0ec 100644 --- a/tests/test_parent_ci.py +++ b/tests/test_parent_ci.py @@ -29,6 +29,19 @@ def test_parent_ci_splits_application_and_act_validation_tests() -> None: REPO_ROOT / ".github" / "workflows" / "act-validation.yml" ) + assert "permissions:\n contents: read" in ci_workflow, ( + "expected parent CI to restrict GITHUB_TOKEN to repository contents reads" + ) + assert "MARKDOWNLINT_CLI2_VERSION: 0.22.1" in ci_workflow, ( + "expected parent CI to pin markdownlint-cli2" + ) + assert "MBAKE_VERSION: 1.4.6" in ci_workflow, "expected parent CI to pin mbake" + assert 'npm install -g "markdownlint-cli2@${MARKDOWNLINT_CLI2_VERSION}"' in ( + ci_workflow + ), "expected parent CI to install pinned markdownlint-cli2" + assert 'uv tool install "mbake==${MBAKE_VERSION}"' in ci_workflow, ( + "expected parent CI to install pinned mbake" + ) assert "make test\n" in ci_workflow, ( "expected parent CI to run the normal parent test gate" ) @@ -41,11 +54,23 @@ def test_parent_ci_splits_application_and_act_validation_tests() -> None: assert "make test WITH_ACT=1" not in ci_workflow, ( "expected parent CI to leave act validation to a separate workflow" ) - assert "ACT_VERSION: v0.2.80" in act_workflow, ( - "expected parent act-validation workflow to pin the act release" + assert "permissions:\n contents: read" in act_workflow, ( + "expected parent act-validation workflow to restrict GITHUB_TOKEN" + ) + assert "ACT_VERSION:" in act_workflow, ( + "expected parent act-validation workflow to declare an act version" + ) + assert ( + "act_Linux_x86_64.tar.gz" in act_workflow and "${ACT_VERSION}" in act_workflow + ), "expected parent act-validation workflow to build the act URL from ACT_VERSION" + assert "sha256sum -c -" in act_workflow, ( + "expected parent act-validation workflow to verify the act archive checksum" ) - assert "act_Linux_x86_64.tar.gz" in act_workflow, ( - "expected parent act-validation workflow to install the act Linux binary" + assert 'npm install -g "markdownlint-cli2@${MARKDOWNLINT_CLI2_VERSION}"' in ( + act_workflow + ), "expected parent act-validation workflow to install pinned markdownlint-cli2" + assert 'uv tool install "mbake==${MBAKE_VERSION}"' in act_workflow, ( + "expected parent act-validation workflow to install pinned mbake" ) assert "uv tool install mdformat-all" not in act_workflow, ( "expected parent act-validation workflow not to install mdformat-all through uv" From 3e88bf1987682ac992576655008d0c69e8b16cf0 Mon Sep 17 00:00:00 2001 From: Payton McIntosh Date: Thu, 4 Jun 2026 19:42:06 +0100 Subject: [PATCH 23/32] Document parent Makefile and CI workflows Add developer-guide coverage for the root Makefile targets, WITH_ACT flag, and separate parent CI workflows for normal template tests and act validation. --- docs/developers-guide.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docs/developers-guide.md b/docs/developers-guide.md index 87154a0..d3e1560 100644 --- a/docs/developers-guide.md +++ b/docs/developers-guide.md @@ -4,6 +4,35 @@ This repository is a Copier template. Source files under `template/` are rendered into generated projects, while tests under `tests/` render those projects and run their public quality gates. +## Parent Repository Makefile + +The root `Makefile` provides developer workflow targets for working on the +template itself. + +- `make help` — lists all `##`-annotated targets. +- `make test` — runs the template test suite via `uvx`, supplying + `pytest-copier`, `pyyaml`, `syrupy`, and `make-parser` without a manually + managed virtual environment. +- `make test WITH_ACT=1` — sets `RUN_ACT_VALIDATION=1` inside the pytest + invocation, enabling the act-backed integration tests that run generated CI + workflows locally. Requires `act` and Docker to be available. + +`uvx` is required; the Makefile aborts with an error if it is not found on +`PATH`. + +## Parent CI Workflows + +The parent repository uses two separate GitHub Actions workflows to keep +Docker-dependent tests isolated from the standard template test gate: + +- `.github/workflows/ci.yml` runs `make test` on every push to `main` and on + all pull requests. It installs `markdownlint-cli2` and `mbake` at pinned + versions. +- `.github/workflows/act-validation.yml` runs `make test WITH_ACT=1`. It + additionally downloads the `act` binary at a pinned `ACT_VERSION`, verifies + its SHA-256 checksum before extraction, and confirms Docker availability via + `docker info`. + ## Makefile Template `template/Makefile.jinja` defines the generated developer workflow. The default From ffbdbe24d8e7362e561805881d9dda97175efe9a Mon Sep 17 00:00:00 2001 From: Payton McIntosh Date: Thu, 4 Jun 2026 19:48:00 +0100 Subject: [PATCH 24/32] Harden generated act validation workflow Mirror the parent act-validation workflow hardening into the rendered template: restrict workflow token permissions, pin markdownlint-cli2 and mbake, and verify the downloaded act archive checksum before extraction. Extend generated workflow contract assertions to cover these protections. --- .../workflows/act-validation.yml.jinja | 18 ++++++++++++++-- tests/helpers/ci_contracts.py | 21 +++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/template/.github/workflows/act-validation.yml.jinja b/template/.github/workflows/act-validation.yml.jinja index 20c4bc5..98bd54d 100644 --- a/template/.github/workflows/act-validation.yml.jinja +++ b/template/.github/workflows/act-validation.yml.jinja @@ -5,11 +5,16 @@ on: branches: [main] pull_request: +permissions: + contents: read + jobs: act-validation: runs-on: ubuntu-latest env: ACT_VERSION: v0.2.80 + MARKDOWNLINT_CLI2_VERSION: 0.22.1 + MBAKE_VERSION: 1.4.6 steps: - name: Check out repository uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 @@ -33,8 +38,17 @@ jobs: {% endif %} - name: Install act run: | - curl -fsSL "https://github.com/nektos/act/releases/download/${ACT_VERSION}/act_Linux_x86_64.tar.gz" | - sudo tar -xz -C /usr/local/bin act + act_archive=act_Linux_x86_64.tar.gz + act_url="https://github.com/nektos/act/releases/download/${ACT_VERSION}/${act_archive}" + curl -fsSLo "${act_archive}" "${act_url}" + act_sha256="$( + curl -fsSL "https://api.github.com/repos/nektos/act/releases/tags/${ACT_VERSION}" | + python -c 'import json, sys; release = json.load(sys.stdin); print(next(asset["digest"].split(":", 1)[1] for asset in release["assets"] if asset["name"] == "act_Linux_x86_64.tar.gz"))' + )" + printf '%s %s\n' "${act_sha256}" "${act_archive}" | sha256sum -c - + sudo tar -xzf "${act_archive}" -C /usr/local/bin act + npm install -g "markdownlint-cli2@${MARKDOWNLINT_CLI2_VERSION}" + uv tool install "mbake==${MBAKE_VERSION}" - name: Check Docker runtime run: docker info diff --git a/tests/helpers/ci_contracts.py b/tests/helpers/ci_contracts.py index 78f71d6..6461aad 100644 --- a/tests/helpers/ci_contracts.py +++ b/tests/helpers/ci_contracts.py @@ -176,9 +176,30 @@ def _assert_act_validation_workflow_contracts( assert "ACT_VERSION: v0.2.80" in act_validation_workflow, ( "expected generated act-validation workflow to pin act" ) + assert "permissions:\n contents: read" in act_validation_workflow, ( + "expected generated act-validation workflow to restrict GITHUB_TOKEN" + ) + assert "MARKDOWNLINT_CLI2_VERSION: 0.22.1" in act_validation_workflow, ( + "expected generated act-validation workflow to pin markdownlint-cli2" + ) + assert "MBAKE_VERSION: 1.4.6" in act_validation_workflow, ( + "expected generated act-validation workflow to pin mbake" + ) assert "act_Linux_x86_64.tar.gz" in act_validation_workflow, ( "expected generated act-validation workflow to install act" ) + assert "${ACT_VERSION}/${act_archive}" in act_validation_workflow, ( + "expected generated act-validation workflow to build the act URL from ACT_VERSION" + ) + assert "sha256sum -c -" in act_validation_workflow, ( + "expected generated act-validation workflow to verify the act archive checksum" + ) + assert 'npm install -g "markdownlint-cli2@${MARKDOWNLINT_CLI2_VERSION}"' in ( + act_validation_workflow + ), "expected generated act-validation workflow to install pinned markdownlint-cli2" + assert 'uv tool install "mbake==${MBAKE_VERSION}"' in act_validation_workflow, ( + "expected generated act-validation workflow to install pinned mbake" + ) assert "docker info" in act_validation_workflow, ( "expected generated act-validation workflow to verify Docker" ) From a280a9abf2bc9a6e2908289cdf7d594edec3edab Mon Sep 17 00:00:00 2001 From: Payton McIntosh Date: Fri, 5 Jun 2026 11:19:55 +0100 Subject: [PATCH 25/32] Document audit in README quality gate flow Add the generated audit target to the README make all flowchart, including pip-audit and cargo audit branches, and expand the caption. Wrap the opening README sentence so markdownlint passes. --- README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e780bdc..3da086d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # Generic Copier Template -This repository provides a [Copier](https://copier.readthedocs.io/) template for a Python package. +This repository provides a [Copier](https://copier.readthedocs.io/) template +for a Python package. It offers two flavours: 1. **Python Only** – a pure Python implementation. @@ -38,7 +39,8 @@ dependencies into the current Python environment manually. Figure: The generated `make all` quality gate runs build, formatting, linting, typechecking, and testing. Rust-specific lint and test branches run only when -the Rust extension is enabled. +the Rust extension is enabled. The `audit` target checks Python dependencies +via `pip-audit` and, when Rust is enabled, Rust dependencies via `cargo audit`. ```mermaid flowchart LR @@ -47,17 +49,20 @@ flowchart LR All --> CheckFmt[check-fmt] All --> Lint[lint] All --> Typecheck[typecheck] + All --> Audit[audit] All --> Test[test] Lint --> RuffCheck[ruff check] Lint --> PylintPyPy[pylint-pypy via PyPy] Typecheck --> TyCheck[ty check] + Audit --> PipAudit[pip-audit] Test --> Pytest[pytest -v -n auto] subgraph Rust_extension_enabled[use_rust == true] Lint --> RustDoc[cargo doc] Lint --> Clippy[cargo clippy] Lint --> Whitaker[whitaker --all] + Audit --> CargoAudit[cargo audit] Test --> Nextest[cargo nextest run] Test --> RustDocTests[cargo test --doc] end From add476719dcc3ca48b3fc3c8496aee34486ff6a8 Mon Sep 17 00:00:00 2001 From: Payton McIntosh Date: Fri, 5 Jun 2026 12:34:37 +0100 Subject: [PATCH 26/32] Clarify workflow contract assertion failures Add an explicit generated Makefile target guard before joining parsed recipes so missing command contracts fail with an actionable assertion instead of KeyError. Split the parent act workflow URL assertion into separate archive and version checks for clearer failure messages. --- tests/helpers/agents_contracts.py | 4 ++++ tests/test_parent_ci.py | 9 ++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/helpers/agents_contracts.py b/tests/helpers/agents_contracts.py index 3ce35c4..bf2e4c5 100644 --- a/tests/helpers/agents_contracts.py +++ b/tests/helpers/agents_contracts.py @@ -164,6 +164,10 @@ def _assert_documented_command_flags( } ) for target, checks in command_contracts.items(): + assert target in makefile_rules, ( + "expected generated Makefile rules to include target " + f"{target!r} from command_contracts" + ) commands = "\n".join(makefile_rules[target]) for source, fragment in checks: haystack = agents if source == "AGENTS.md" else commands diff --git a/tests/test_parent_ci.py b/tests/test_parent_ci.py index e26b0ec..1e25372 100644 --- a/tests/test_parent_ci.py +++ b/tests/test_parent_ci.py @@ -60,9 +60,12 @@ def test_parent_ci_splits_application_and_act_validation_tests() -> None: assert "ACT_VERSION:" in act_workflow, ( "expected parent act-validation workflow to declare an act version" ) - assert ( - "act_Linux_x86_64.tar.gz" in act_workflow and "${ACT_VERSION}" in act_workflow - ), "expected parent act-validation workflow to build the act URL from ACT_VERSION" + assert "act_Linux_x86_64.tar.gz" in act_workflow, ( + "expected parent act-validation workflow to include act_Linux_x86_64.tar.gz" + ) + assert "${ACT_VERSION}" in act_workflow, ( + "expected parent act-validation workflow to include ${ACT_VERSION}" + ) assert "sha256sum -c -" in act_workflow, ( "expected parent act-validation workflow to verify the act archive checksum" ) From 22de12b2b8e42145ee251ec2993f83a60ee0d8a5 Mon Sep 17 00:00:00 2001 From: Payton McIntosh Date: Fri, 5 Jun 2026 18:14:08 +0100 Subject: [PATCH 27/32] Clear RUSTFLAGS during generated Rust tool installs Add a step-level RUSTFLAGS override to the generated Rust CI tool installation step so host-inherited -D warnings does not break cargo install for third-party tools under act. Keep project build and test steps unaffected. Assert the rendered Rust workflow contains the override. --- template/.github/workflows/ci.yml.jinja | 2 ++ tests/helpers/ci_contracts.py | 17 +++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/template/.github/workflows/ci.yml.jinja b/template/.github/workflows/ci.yml.jinja index 6c057de..fd59b8c 100644 --- a/template/.github/workflows/ci.yml.jinja +++ b/template/.github/workflows/ci.yml.jinja @@ -47,6 +47,8 @@ jobs: ~/.local/share/whitaker - name: Install Rust lint and test tools + env: + RUSTFLAGS: "" run: | cargo install --locked \ --git https://github.com/leynos/whitaker \ diff --git a/tests/helpers/ci_contracts.py b/tests/helpers/ci_contracts.py index 6461aad..aa3860c 100644 --- a/tests/helpers/ci_contracts.py +++ b/tests/helpers/ci_contracts.py @@ -145,6 +145,23 @@ def _assert_ci_workflow_contracts( assert "leynos/shared-actions/.github/actions/setup-rust" in ci_workflow, ( "expected Rust variant CI to set up Rust" ) + rust_tool_steps = [ + step + for step in steps + if isinstance(step, dict) + and step.get("name") == "Install Rust lint and test tools" + ] + assert len(rust_tool_steps) == 1, ( + "expected Rust variant CI to install Rust lint and test tools once" + ) + rust_tool_env = require_mapping( + rust_tool_steps[0], + "env", + "Install Rust lint and test tools step", + ) + assert rust_tool_env.get("RUSTFLAGS") == "", ( + "expected Rust tool installation step to clear inherited RUSTFLAGS" + ) assert "Cache Rust lint and test tools" in ci_workflow, ( "expected Rust variant CI to cache Rust tools" ) From e235e3a129811f7f4067178d72391a6ef9409a1e Mon Sep 17 00:00:00 2001 From: Payton McIntosh Date: Fri, 5 Jun 2026 18:44:38 +0100 Subject: [PATCH 28/32] Report act coverage failures after log checks Check act logs for coverage and test execution before requiring coverage.xml on disk. This lets the known act composite-output/archive xfail apply after tests and coverage have been observed, instead of failing early with a misleading coverage-file assertion. --- tests/test_github_actions_integration.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/test_github_actions_integration.py b/tests/test_github_actions_integration.py index ee22d0c..8296d99 100644 --- a/tests/test_github_actions_integration.py +++ b/tests/test_github_actions_integration.py @@ -293,12 +293,7 @@ def assert_act_result( assert_act_result(project, code, logs, use_rust=False) """ - assert (project / "coverage.xml").exists(), ( - "act workflow should write coverage.xml in the generated project" - ) assert_ci_exercised_expected_steps(logs, use_rust=use_rust) - if code == 0: - return if ( "Parameter INPUT_ARTEFACT_NAME_SUFFIX specified multiple times" in logs and "Provided artifact name input during validation is empty" in logs @@ -308,6 +303,9 @@ def assert_act_result( "action output/archive phase after tests and coverage succeed" ) assert code == 0, logs + assert (project / "coverage.xml").exists(), ( + "act workflow should write coverage.xml in the generated project" + ) @pytest.mark.parametrize( From 67cf11632a0134d93c6534821ffd8bbd53f15f39 Mon Sep 17 00:00:00 2001 From: Payton McIntosh Date: Fri, 5 Jun 2026 19:06:31 +0100 Subject: [PATCH 29/32] Document temporary lint suppression follow-ups Require temporary lint suppressions in rendered AGENTS.md to link to the planned fix or implementation that will remove the suppression. Add a generated AGENTS contract assertion so the guidance does not regress. --- template/AGENTS.md.jinja | 2 ++ tests/helpers/agents_contracts.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/template/AGENTS.md.jinja b/template/AGENTS.md.jinja index a71f9d8..05637bc 100644 --- a/template/AGENTS.md.jinja +++ b/template/AGENTS.md.jinja @@ -87,6 +87,8 @@ - `make test` runs `pytest -v -n $(PYTEST_XDIST_WORKERS)` and honours `WITH_ACT=1` through `RUN_ACT_VALIDATION=1`. - `make audit` runs `pip-audit`. + - Temporary lint suppressions must include a link to the planned fix or + implementation that will remove the suppression. {% if use_rust -%} - For Rust files: - **Testing:** Passes relevant unit and behavioural tests (`make test`). diff --git a/tests/helpers/agents_contracts.py b/tests/helpers/agents_contracts.py index bf2e4c5..8004e6a 100644 --- a/tests/helpers/agents_contracts.py +++ b/tests/helpers/agents_contracts.py @@ -35,6 +35,10 @@ def _assert_agents_contracts(agents: str) -> None: assert "brittle snapshots" in agents, ( "expected generated AGENTS.md to warn against brittle snapshot churn" ) + assert "Temporary lint suppressions must include a link" in agents, ( + "expected generated AGENTS.md to require linked plans for temporary " + "lint suppressions" + ) def _assert_agents_make_targets_mirror_makefile( From 60ebe6ab625e68a9392ef29c78a5d18444a4c62f Mon Sep 17 00:00:00 2001 From: Payton McIntosh Date: Fri, 5 Jun 2026 19:43:07 +0100 Subject: [PATCH 30/32] Update shared actions coverage pin Pin parent and rendered workflows to shared-actions commit 455d9ed. Constrain generated pytest discovery to the tests tree for xdist-backed SlipCover coverage. Add parent Makefile gate targets and contracts so check-fmt, lint, typecheck, and test all run from the repository root. --- .github/workflows/act-validation.yml | 2 +- .github/workflows/ci.yml | 2 +- Makefile | 11 +++++- README.md | 7 ++-- docs/developers-guide.md | 10 ++++++ .../workflows/act-validation.yml.jinja | 2 +- template/.github/workflows/ci.yml.jinja | 4 +-- template/.rules/python-00.md | 12 +++---- template/AGENTS.md.jinja | 3 ++ template/docs/users-guide.md.jinja | 5 +++ template/pyproject.toml.jinja | 1 + tests/helpers/agents_contracts.py | 4 +++ tests/helpers/ci_contracts.py | 2 +- tests/helpers/pyproject_contracts.py | 13 +++++++ tests/test_helpers.py | 34 +++++++++++++++---- 15 files changed, 89 insertions(+), 23 deletions(-) diff --git a/.github/workflows/act-validation.yml b/.github/workflows/act-validation.yml index da90edb..16c1c31 100644 --- a/.github/workflows/act-validation.yml +++ b/.github/workflows/act-validation.yml @@ -30,7 +30,7 @@ jobs: uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 - name: Set up Rust - uses: leynos/shared-actions/.github/actions/setup-rust@d400b079fb6a8fa92f7e7b6c57f3d1c92a4b2d54 + uses: leynos/shared-actions/.github/actions/setup-rust@455d9ed03477c0026da96c2541ca26569a74acac with: toolchain: stable diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8f4f801..59df0fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,7 @@ jobs: uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 - name: Set up Rust - uses: leynos/shared-actions/.github/actions/setup-rust@d400b079fb6a8fa92f7e7b6c57f3d1c92a4b2d54 + uses: leynos/shared-actions/.github/actions/setup-rust@455d9ed03477c0026da96c2541ca26569a74acac with: toolchain: stable diff --git a/Makefile b/Makefile index df34ed6..3355b8c 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help test +.PHONY: help check-fmt lint test typecheck MAKEFLAGS += --no-print-directory @@ -13,6 +13,15 @@ endif test: ## Run template tests $(ACT_TEST_ENV) $(UV) --with pytest-copier --with pyyaml --with syrupy --with make-parser pytest tests/ +check-fmt: ## Verify template test formatting + $(UV) ruff format --check tests/ + +lint: ## Run template test lint checks + $(UV) ruff check tests/ + +typecheck: ## Run template test type checks + $(UV) --with pytest --with pytest-copier --with pyyaml --with syrupy --with make-parser ty check tests/ + help: ## Show available targets @grep -E '^[a-zA-Z_-]+:.*?##' $(MAKEFILE_LIST) | \ awk 'BEGIN {FS=":.*?## "; printf "Available targets:\n"} {printf " %-15s %s\n", $$1, $$2}' diff --git a/README.md b/README.md index 3da086d..f4db656 100644 --- a/README.md +++ b/README.md @@ -16,9 +16,10 @@ projects that run Ruff, Pylint via a PyPy-backed runner, `ty`, pytest, and, when the Rust extension is enabled, Clippy, Whitaker, and nextest-aware Rust tests. Run the parent template tests through the repository `Makefile`. Run -`make help` to list the available parent Makefile targets. The `test` target -uses `uvx` to provide `pytest-copier`, `pyyaml`, `syrupy`, and `make-parser` -without a manually managed virtual environment: +`make help` to list the available parent Makefile targets. Parent gates include +`make check-fmt`, `make lint`, `make typecheck`, and `make test`. The `test` +target uses `uvx` to provide `pytest-copier`, `pyyaml`, `syrupy`, and +`make-parser` without a manually managed virtual environment: ```bash make test diff --git a/docs/developers-guide.md b/docs/developers-guide.md index d3e1560..202c52e 100644 --- a/docs/developers-guide.md +++ b/docs/developers-guide.md @@ -10,6 +10,11 @@ The root `Makefile` provides developer workflow targets for working on the template itself. - `make help` — lists all `##`-annotated targets. +- `make check-fmt` — runs Ruff formatting checks against the parent template + test suite. +- `make lint` — runs Ruff lint checks against the parent template test suite. +- `make typecheck` — runs `ty check` against the parent template test suite, + supplying the test dependencies through `uvx`. - `make test` — runs the template test suite via `uvx`, supplying `pytest-copier`, `pyyaml`, `syrupy`, and `make-parser` without a manually managed virtual environment. @@ -61,6 +66,11 @@ sets up Python, optionally sets up Rust, runs `make check-fmt`, `make lint`, `make typecheck`, and `make audit`, then delegates coverage to `leynos/shared-actions/.github/actions/generate-coverage`. +The shared coverage action runs Python coverage through xdist-backed SlipCover +support. Generated pytest discovery is therefore constrained to the top-level +`tests/` tree via `tool.pytest.ini_options.testpaths`; do not add pytest unit +tests under package module directories or `unittests/` subdirectories. + `template/.github/workflows/act-validation.yml.jinja` keeps rendered workflow validation separate from generated application CI. It installs `act`, verifies Docker availability, and runs `make test WITH_ACT=1`. diff --git a/template/.github/workflows/act-validation.yml.jinja b/template/.github/workflows/act-validation.yml.jinja index 98bd54d..36a33e1 100644 --- a/template/.github/workflows/act-validation.yml.jinja +++ b/template/.github/workflows/act-validation.yml.jinja @@ -31,7 +31,7 @@ jobs: {% if use_rust %} - name: Set up Rust - uses: leynos/shared-actions/.github/actions/setup-rust@d400b079fb6a8fa92f7e7b6c57f3d1c92a4b2d54 + uses: leynos/shared-actions/.github/actions/setup-rust@455d9ed03477c0026da96c2541ca26569a74acac with: toolchain: stable diff --git a/template/.github/workflows/ci.yml.jinja b/template/.github/workflows/ci.yml.jinja index fd59b8c..c9ec18a 100644 --- a/template/.github/workflows/ci.yml.jinja +++ b/template/.github/workflows/ci.yml.jinja @@ -32,7 +32,7 @@ jobs: {% if use_rust %} - name: Set up Rust - uses: leynos/shared-actions/.github/actions/setup-rust@d400b079fb6a8fa92f7e7b6c57f3d1c92a4b2d54 + uses: leynos/shared-actions/.github/actions/setup-rust@455d9ed03477c0026da96c2541ca26569a74acac with: toolchain: stable @@ -82,7 +82,7 @@ jobs: run: make audit - name: Test and Measure Coverage - uses: leynos/shared-actions/.github/actions/generate-coverage@d400b079fb6a8fa92f7e7b6c57f3d1c92a4b2d54 + uses: leynos/shared-actions/.github/actions/generate-coverage@455d9ed03477c0026da96c2541ca26569a74acac with: output-path: coverage.xml format: cobertura diff --git a/template/.rules/python-00.md b/template/.rules/python-00.md index 5e3d3ce..6ff106d 100644 --- a/template/.rules/python-00.md +++ b/template/.rules/python-00.md @@ -75,14 +75,14 @@ def scale(values: list[float], factor: float) -> list[float]: ## Testing with pytest -- **Colocate unit tests with code** using an `unittests` subdirectory and a - `test_` prefix. This keeps logic and its tests together: +- **Keep tests in the top-level `tests/` tree.** Do not place pytest unit tests + in package module directories or `unittests/` subdirectories. Coverage runs + through xdist-backed SlipCover support, which currently requires tests to be + outside the import package tree. ```text -user_auth/ - models.py - login_flow.py - unittests/ +tests/ + user_auth/ test_models.py test_login_flow.py ``` diff --git a/template/AGENTS.md.jinja b/template/AGENTS.md.jinja index 05637bc..bb6c5e4 100644 --- a/template/AGENTS.md.jinja +++ b/template/AGENTS.md.jinja @@ -144,6 +144,9 @@ - For Python work, use `pytest` for unit tests and `pytest-bdd` for behavioural tests. Cover happy paths, unhappy paths, and relevant edge cases. +- Keep pytest tests in the top-level `tests/` tree. Do not place unit tests in + package module directories or `unittests/` subdirectories, because CI coverage + uses xdist-backed SlipCover support. - Snapshot tests (using `syrupy`) should be provided where multivariant output format consistency is relevant to the requirements. Snapshot tests must capture meaningful, reviewer-useful contracts rather than generic dumps of diff --git a/template/docs/users-guide.md.jinja b/template/docs/users-guide.md.jinja index c73d131..2b9e639 100644 --- a/template/docs/users-guide.md.jinja +++ b/template/docs/users-guide.md.jinja @@ -18,6 +18,11 @@ The `lint-python` target runs Ruff followed by Pylint via a PyPy-backed runner. The Pylint runner is installed through `uv tool run` from the pinned `pylint-pypy-shim` repository. +Pytest discovery is limited to the top-level `tests/` tree. Keep generated +project unit tests there rather than in package module directories or +`unittests/` subdirectories, because CI coverage runs through xdist-backed +SlipCover support. + When the Rust extension is enabled, `lint-rust` runs: - `cargo doc` with warnings denied; diff --git a/template/pyproject.toml.jinja b/template/pyproject.toml.jinja index c7477bc..6689f87 100644 --- a/template/pyproject.toml.jinja +++ b/template/pyproject.toml.jinja @@ -296,6 +296,7 @@ enable = [ [tool.pytest.ini_options] # Tests automatically killed after seconds elapsed timeout = 30 +testpaths = ["tests"] [tool.uv] package = true diff --git a/tests/helpers/agents_contracts.py b/tests/helpers/agents_contracts.py index 8004e6a..947108b 100644 --- a/tests/helpers/agents_contracts.py +++ b/tests/helpers/agents_contracts.py @@ -39,6 +39,10 @@ def _assert_agents_contracts(agents: str) -> None: "expected generated AGENTS.md to require linked plans for temporary " "lint suppressions" ) + assert "Keep pytest tests in the top-level `tests/` tree" in agents, ( + "expected generated AGENTS.md to keep pytest discovery compatible with " + "xdist-backed coverage" + ) def _assert_agents_make_targets_mirror_makefile( diff --git a/tests/helpers/ci_contracts.py b/tests/helpers/ci_contracts.py index aa3860c..3c5cdae 100644 --- a/tests/helpers/ci_contracts.py +++ b/tests/helpers/ci_contracts.py @@ -71,7 +71,7 @@ def assert_ci_coverage_action_contract( assert ( coverage_step.get("uses") == "leynos/shared-actions/.github/actions/generate-coverage" - "@d400b079fb6a8fa92f7e7b6c57f3d1c92a4b2d54" + "@455d9ed03477c0026da96c2541ca26569a74acac" ), "expected CI to use the pinned shared coverage action" coverage_inputs = require_mapping(coverage_step, "with", "coverage step") assert coverage_inputs.get("output-path") == "coverage.xml", ( diff --git a/tests/helpers/pyproject_contracts.py b/tests/helpers/pyproject_contracts.py index 8ed8408..7e32024 100644 --- a/tests/helpers/pyproject_contracts.py +++ b/tests/helpers/pyproject_contracts.py @@ -36,6 +36,19 @@ def _assert_pyproject_contracts( assert dependency in dev_dependencies, ( f"expected generated dev dependencies to include {dependency}" ) + pytest_options = require_mapping( + require_mapping(pyproject, "tool", "pyproject.toml"), + "pytest", + "pyproject.toml tool", + ) + pytest_ini_options = require_mapping( + pytest_options, + "ini_options", + "pyproject.toml tool.pytest", + ) + assert pytest_ini_options.get("testpaths") == ["tests"], ( + "expected generated pytest discovery to be limited to the tests tree" + ) build_system = require_mapping(pyproject, "build-system", "pyproject.toml") if use_rust: diff --git a/tests/test_helpers.py b/tests/test_helpers.py index f918c3f..256625b 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -300,7 +300,7 @@ def test_parent_makefile_help_target_lists_available_targets() -> None: ------- None The test passes when ``make help`` advertises the parent ``help`` and - ``test`` targets. + quality-gate targets. """ result = subprocess.run( ["make", "help"], @@ -316,9 +316,10 @@ def test_parent_makefile_help_target_lists_available_targets() -> None: assert "help" in result.stdout, ( "expected parent Makefile help target to list the help target" ) - assert "test" in result.stdout, ( - "expected parent Makefile help target to list the test target" - ) + for target in ["check-fmt", "lint", "typecheck", "test"]: + assert target in result.stdout, ( + f"expected parent Makefile help target to list the {target} target" + ) def test_parent_makefile_test_target_uses_requisite_pytest_command() -> None: @@ -338,9 +339,28 @@ def test_parent_makefile_test_target_uses_requisite_pytest_command() -> None: """ makefile = (REPO_ROOT / "Makefile").read_text(encoding="utf-8") - assert ".PHONY: help test" in makefile, ( - "expected parent Makefile to mark help and test as phony targets" + assert ".PHONY: help check-fmt lint test typecheck" in makefile, ( + "expected parent Makefile to mark documented gate targets as phony" + ) + assert "check-fmt: ## Verify template test formatting" in makefile, ( + "expected parent Makefile to expose a documented check-fmt target" + ) + assert "$(UV) ruff format --check tests/" in makefile, ( + "expected parent Makefile check-fmt target to run Ruff formatting checks" ) + assert "lint: ## Run template test lint checks" in makefile, ( + "expected parent Makefile to expose a documented lint target" + ) + assert "$(UV) ruff check tests/" in makefile, ( + "expected parent Makefile lint target to run Ruff checks" + ) + assert "typecheck: ## Run template test type checks" in makefile, ( + "expected parent Makefile to expose a documented typecheck target" + ) + assert ( + "$(UV) --with pytest --with pytest-copier --with pyyaml --with syrupy " + "--with make-parser ty check tests/" in makefile + ), "expected parent Makefile typecheck target to run ty with test dependencies" assert "test: ## Run template tests" in makefile, ( "expected parent Makefile to expose a documented test target" ) @@ -376,7 +396,7 @@ def _ci_workflow(*, persist_credentials: str, coverage_inputs: str) -> str: with: persist-credentials: {persist_credentials} - name: Test and Measure Coverage - uses: leynos/shared-actions/.github/actions/generate-coverage@d400b079fb6a8fa92f7e7b6c57f3d1c92a4b2d54 + uses: leynos/shared-actions/.github/actions/generate-coverage@455d9ed03477c0026da96c2541ca26569a74acac with: output-path: coverage.xml format: cobertura From e480d721542b731133e0c986810ae43780107cd0 Mon Sep 17 00:00:00 2001 From: Payton McIntosh Date: Fri, 5 Jun 2026 21:36:45 +0100 Subject: [PATCH 31/32] Require Node24-capable act for validation Upgrade parent and rendered act-validation workflows to act v0.2.84 so act supports Node24 JavaScript actions. Add a local act version preflight that skips optional act-backed tests when the installed binary is too old. Scrub host GitHub auth tokens from act subprocess environments so stale local credentials do not break public action clones. --- .github/workflows/act-validation.yml | 2 +- .../workflows/act-validation.yml.jinja | 2 +- tests/conftest.py | 42 ++++++++ tests/helpers/ci_contracts.py | 2 +- tests/test_helpers.py | 102 ++++++++++++++++++ tests/utilities.py | 8 +- 6 files changed, 154 insertions(+), 4 deletions(-) diff --git a/.github/workflows/act-validation.yml b/.github/workflows/act-validation.yml index 16c1c31..cc4b58b 100644 --- a/.github/workflows/act-validation.yml +++ b/.github/workflows/act-validation.yml @@ -12,7 +12,7 @@ jobs: act-validation: runs-on: ubuntu-latest env: - ACT_VERSION: v0.2.80 + ACT_VERSION: v0.2.84 MARKDOWNLINT_CLI2_VERSION: 0.22.1 MBAKE_VERSION: 1.4.6 steps: diff --git a/template/.github/workflows/act-validation.yml.jinja b/template/.github/workflows/act-validation.yml.jinja index 36a33e1..c76ea3c 100644 --- a/template/.github/workflows/act-validation.yml.jinja +++ b/template/.github/workflows/act-validation.yml.jinja @@ -12,7 +12,7 @@ jobs: act-validation: runs-on: ubuntu-latest env: - ACT_VERSION: v0.2.80 + ACT_VERSION: v0.2.84 MARKDOWNLINT_CLI2_VERSION: 0.22.1 MBAKE_VERSION: 1.4.6 steps: diff --git a/tests/conftest.py b/tests/conftest.py index 4f21e9b..8a17760 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -24,6 +24,7 @@ from __future__ import annotations import os +import re import shutil import subprocess @@ -31,6 +32,9 @@ from tests.utilities import docker_environment +MINIMUM_ACT_VERSION = (0, 2, 84) +ACT_VERSION_PATTERN = re.compile(r"\bact version v?(\d+)\.(\d+)\.(\d+)\b") + @pytest.fixture(scope="session") def copier_template_paths() -> list[str]: @@ -110,6 +114,36 @@ def _runtime_info_commands() -> list[list[str]]: return commands +def _parse_act_version(output: str) -> tuple[int, int, int] | None: + """Return the semantic act version from command output.""" + match = ACT_VERSION_PATTERN.search(output) + if match is None: + return None + major, minor, patch = match.groups() + return int(major), int(minor), int(patch) + + +def _installed_act_version() -> tuple[int, int, int]: + """Return the installed act version or skip when it cannot be parsed.""" + try: + version = subprocess.run( + ["act", "--version"], + text=True, + capture_output=True, + check=False, + timeout=30, + ) + except (OSError, subprocess.SubprocessError) as exc: + pytest.skip(f"could not determine act version: {exc}") + parsed_version = _parse_act_version(version.stdout) + if parsed_version is None: + pytest.skip( + "could not parse act version from 'act --version' output: " + f"{version.stdout.strip() or version.stderr.strip()}" + ) + return parsed_version + + @pytest.fixture def act_ready() -> None: """Skip act-backed tests unless local workflow validation can run. @@ -145,6 +179,14 @@ def test_workflow(act_ready: None) -> None: pytest.skip("set RUN_ACT_VALIDATION=1 to run act workflow validation") if shutil.which("act") is None: pytest.skip("act is not installed") + act_version = _installed_act_version() + if act_version < MINIMUM_ACT_VERSION: + found_version = ".".join(str(part) for part in act_version) + minimum_version = ".".join(str(part) for part in MINIMUM_ACT_VERSION) + pytest.skip( + f"act >= {minimum_version} is required for Node24 action runtime " + f"support; found act version {found_version}" + ) info_commands = _runtime_info_commands() if not info_commands: pytest.skip("docker-compatible container runtime is not installed") diff --git a/tests/helpers/ci_contracts.py b/tests/helpers/ci_contracts.py index 3c5cdae..2226dec 100644 --- a/tests/helpers/ci_contracts.py +++ b/tests/helpers/ci_contracts.py @@ -190,7 +190,7 @@ def _assert_act_validation_workflow_contracts( assert "name: Act Validation" in act_validation_workflow, ( "expected generated act-validation workflow to be named" ) - assert "ACT_VERSION: v0.2.80" in act_validation_workflow, ( + assert "ACT_VERSION: v0.2.84" in act_validation_workflow, ( "expected generated act-validation workflow to pin act" ) assert "permissions:\n contents: read" in act_validation_workflow, ( diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 256625b..976a7fe 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -14,6 +14,7 @@ import pytest +from tests.conftest import MINIMUM_ACT_VERSION, _parse_act_version from tests.helpers.generated_files import ( parse_toml_file, parse_yaml_mapping, @@ -26,6 +27,7 @@ assert_ci_coverage_action_contract, assert_common_make_targets, ) +from tests.utilities import docker_environment if TYPE_CHECKING: from pytest_copier.plugin import CopierProject @@ -34,6 +36,106 @@ REPO_ROOT = Path(__file__).resolve().parent.parent +@pytest.mark.parametrize( + ("output", "expected"), + [ + ("act version 0.2.84", (0, 2, 84)), + ("act version v0.2.84", (0, 2, 84)), + ], +) +def test_parse_act_version_accepts_supported_formats( + output: str, + expected: tuple[int, int, int], +) -> None: + """Parse act version output used by the local preflight. + + Parameters + ---------- + output + Text emitted by ``act --version``. + expected + Semantic version tuple expected from the parser. + + Returns + ------- + None + The test passes when supported act version output formats parse to the + expected tuple. + """ + assert _parse_act_version(output) == expected, ( + f"expected act version parser to parse {output!r}" + ) + + +def test_parse_act_version_rejects_unexpected_output() -> None: + """Reject act version output that does not contain a semantic version. + + Parameters + ---------- + None + This test does not use pytest fixtures. + + Returns + ------- + None + The test passes when unexpected output returns ``None`` so the preflight + can skip optional act-backed tests with a clear reason. + """ + assert _parse_act_version("unexpected act output") is None, ( + "expected act version parser to reject output without a semantic version" + ) + + +def test_old_act_version_is_below_minimum() -> None: + """Compare stale act versions against the Node24-capable minimum. + + Parameters + ---------- + None + This test does not use pytest fixtures. + + Returns + ------- + None + The test passes when act ``0.2.80`` is detected as older than the + minimum version required for Node24 action runtime support. + """ + parsed_version = _parse_act_version("act version 0.2.80") + + assert parsed_version is not None, "expected parser to read stale act version" + assert parsed_version < MINIMUM_ACT_VERSION, ( + "expected act 0.2.80 to be below the Node24-capable minimum" + ) + + +def test_docker_environment_removes_github_auth_tokens( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Remove host GitHub credentials from act subprocess environments. + + Parameters + ---------- + monkeypatch + Pytest fixture used to install representative host GitHub token + variables. + + Returns + ------- + None + The test passes when ``docker_environment`` removes GitHub auth tokens + so stale host credentials cannot break public action clones in ``act``. + """ + monkeypatch.setenv("GITHUB_TOKEN", "stale-token") + monkeypatch.setenv("GH_TOKEN", "stale-token") + + env = docker_environment() + + assert "GITHUB_TOKEN" not in env, ( + "expected docker_environment to remove host GITHUB_TOKEN" + ) + assert "GH_TOKEN" not in env, "expected docker_environment to remove host GH_TOKEN" + + def test_read_generated_text_converts_os_errors(tmp_path: Path) -> None: """Convert generated-file read errors into pytest failures. diff --git a/tests/utilities.py b/tests/utilities.py index f854df4..29a7fc7 100644 --- a/tests/utilities.py +++ b/tests/utilities.py @@ -15,7 +15,9 @@ ----------- Only canonical local Unix socket paths are accepted. Remote Docker endpoints, malformed ``DOCKER_HOST`` values, missing sockets, and sockets outside the -allowed runtime directories are removed or ignored. +allowed runtime directories are removed or ignored. Host GitHub authentication +tokens are removed so stale local credentials cannot break public action clones +inside ``act``. Examples -------- @@ -31,6 +33,8 @@ from pathlib import Path from urllib.parse import urlparse +GITHUB_AUTH_ENV_VARS = ("GITHUB_TOKEN", "GH_TOKEN") + def _resolved_socket_from_docker_host( docker_host: str, allowed_dirs: tuple[Path, ...] @@ -150,6 +154,8 @@ def docker_environment() -> dict[str, str]: subprocess.run(command, env=docker_environment(), check=False) """ env = os.environ.copy() + for variable in GITHUB_AUTH_ENV_VARS: + env.pop(variable, None) docker_host = env.get("DOCKER_HOST") if docker_host is not None: socket_path = _resolved_socket_from_docker_host( From b44b9a3de3e503da30ebe88598a55b4ef4ea9f5d Mon Sep 17 00:00:00 2001 From: Payton McIntosh Date: Sat, 6 Jun 2026 01:12:42 +0100 Subject: [PATCH 32/32] Forward nested act GitHub token explicitly Expose ACT_GITHUB_TOKEN only on the parent act-validation test step and forward it to nested act as the GITHUB_TOKEN secret. Keep ambient GITHUB_TOKEN and GH_TOKEN sanitisation unchanged so stale local credentials are not inherited accidentally. Add unit coverage for both the no-secret and explicit-secret run_act command paths. --- .github/workflows/act-validation.yml | 2 + docs/developers-guide.md | 4 +- tests/test_github_actions_integration.py | 75 ++++++++++++++++++++++++ tests/test_parent_ci.py | 4 ++ 4 files changed, 84 insertions(+), 1 deletion(-) diff --git a/.github/workflows/act-validation.yml b/.github/workflows/act-validation.yml index cc4b58b..2531ef7 100644 --- a/.github/workflows/act-validation.yml +++ b/.github/workflows/act-validation.yml @@ -52,4 +52,6 @@ jobs: run: docker info - name: Run template tests with act validation + env: + ACT_GITHUB_TOKEN: ${{ github.token }} run: make test WITH_ACT=1 diff --git a/docs/developers-guide.md b/docs/developers-guide.md index 202c52e..1c91f98 100644 --- a/docs/developers-guide.md +++ b/docs/developers-guide.md @@ -36,7 +36,9 @@ Docker-dependent tests isolated from the standard template test gate: - `.github/workflows/act-validation.yml` runs `make test WITH_ACT=1`. It additionally downloads the `act` binary at a pinned `ACT_VERSION`, verifies its SHA-256 checksum before extraction, and confirms Docker availability via - `docker info`. + `docker info`. The workflow exposes `ACT_GITHUB_TOKEN: ${{ github.token }}` + only to the nested act test step so actions requiring `github.token` behave + like they do on GitHub-hosted runners. ## Makefile Template diff --git a/tests/test_github_actions_integration.py b/tests/test_github_actions_integration.py index 8296d99..0a840ad 100644 --- a/tests/test_github_actions_integration.py +++ b/tests/test_github_actions_integration.py @@ -26,6 +26,8 @@ import json import subprocess from pathlib import Path +from types import SimpleNamespace +from typing import Any, cast import pytest from pytest_copier.plugin import CopierFixture, CopierProject @@ -119,6 +121,9 @@ def run_act(project: CopierProject, *, artifact_dir: Path) -> tuple[int, str]: docker_host = container_daemon_socket(env) if docker_host is not None: command.extend(["--container-daemon-socket", docker_host]) + act_github_token = env.get("ACT_GITHUB_TOKEN") + if act_github_token: + command.extend(["-s", f"GITHUB_TOKEN={act_github_token}"]) completed = subprocess.run( command, cwd=project.path, @@ -131,6 +136,76 @@ def run_act(project: CopierProject, *, artifact_dir: Path) -> tuple[int, str]: return completed.returncode, f"{completed.stdout}\n{completed.stderr}" +@pytest.mark.parametrize( + ("env", "expected_secret"), + [ + ({}, None), + ({"ACT_GITHUB_TOKEN": "nested-token"}, "GITHUB_TOKEN=nested-token"), + ], +) +def test_run_act_forwards_only_explicit_act_github_token( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + env: dict[str, str], + expected_secret: str | None, +) -> None: + """Forward only explicit nested-act GitHub tokens. + + Parameters + ---------- + monkeypatch + Pytest fixture used to replace subprocess and environment helpers. + tmp_path + Temporary directory used as the fake rendered project and artifact + location. + env + Sanitized act subprocess environment returned by ``docker_environment``. + expected_secret + Expected ``GITHUB_TOKEN`` secret argument, or ``None`` when no secret + should be passed to act. + + Returns + ------- + None + The test passes when ``run_act`` forwards ``ACT_GITHUB_TOKEN`` as an act + secret and does not synthesize a secret when it is absent. + """ + captured_command: list[str] = [] + + def fake_run(command: list[str], **_: Any) -> subprocess.CompletedProcess[str]: + captured_command.extend(command) + return subprocess.CompletedProcess(command, 0, "stdout", "stderr") + + monkeypatch.setattr( + "tests.test_github_actions_integration.docker_environment", + lambda: env, + ) + monkeypatch.setattr( + "tests.test_github_actions_integration.container_daemon_socket", + lambda _: None, + ) + monkeypatch.setattr(subprocess, "run", fake_run) + project = cast("CopierProject", SimpleNamespace(path=tmp_path)) + + run_act(project, artifact_dir=tmp_path / "artifacts") + + if expected_secret is None: + assert "-s" not in captured_command, ( + "expected run_act not to pass act secrets without ACT_GITHUB_TOKEN" + ) + assert not any( + argument.startswith("GITHUB_TOKEN=") for argument in captured_command + ), "expected run_act not to synthesize a GITHUB_TOKEN secret" + else: + assert "-s" in captured_command, ( + "expected run_act to pass an act secret when ACT_GITHUB_TOKEN is set" + ) + secret_index = captured_command.index("-s") + 1 + assert captured_command[secret_index] == expected_secret, ( + "expected run_act to forward ACT_GITHUB_TOKEN as GITHUB_TOKEN secret" + ) + + def iter_json_log_events(logs: str) -> list[dict[str, object]]: """Return JSON event objects from an act log stream. diff --git a/tests/test_parent_ci.py b/tests/test_parent_ci.py index 1e25372..19aaf2c 100644 --- a/tests/test_parent_ci.py +++ b/tests/test_parent_ci.py @@ -84,6 +84,10 @@ def test_parent_ci_splits_application_and_act_validation_tests() -> None: assert "docker info" in act_workflow, ( "expected parent act-validation workflow to verify Docker before act tests" ) + assert "ACT_GITHUB_TOKEN: ${{ github.token }}" in act_workflow, ( + "expected parent act-validation workflow to expose github.token only to " + "nested act tests" + ) assert "make test WITH_ACT=1" in act_workflow, ( "expected parent act-validation workflow to run parent tests with act enabled" )