diff --git a/.github/workflows/act-validation.yml b/.github/workflows/act-validation.yml new file mode 100644 index 0000000..2531ef7 --- /dev/null +++ b/.github/workflows/act-validation.yml @@ -0,0 +1,57 @@ +name: Act Validation + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +jobs: + act-validation: + runs-on: ubuntu-latest + env: + ACT_VERSION: v0.2.84 + MARKDOWNLINT_CLI2_VERSION: 0.22.1 + MBAKE_VERSION: 1.4.6 + steps: + - name: Check out repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: '3.13' + + - name: Install uv + uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 + + - name: Set up Rust + uses: leynos/shared-actions/.github/actions/setup-rust@455d9ed03477c0026da96c2541ca26569a74acac + with: + toolchain: stable + + - name: Install template test tools + run: | + act_archive=act_Linux_x86_64.tar.gz + act_url="https://github.com/nektos/act/releases/download/${ACT_VERSION}/${act_archive}" + curl -fsSLo "${act_archive}" "${act_url}" + act_sha256="$( + curl -fsSL "https://api.github.com/repos/nektos/act/releases/tags/${ACT_VERSION}" | + python -c 'import json, sys; release = json.load(sys.stdin); print(next(asset["digest"].split(":", 1)[1] for asset in release["assets"] if asset["name"] == "act_Linux_x86_64.tar.gz"))' + )" + printf '%s %s\n' "${act_sha256}" "${act_archive}" | sha256sum -c - + sudo tar -xzf "${act_archive}" -C /usr/local/bin act + npm install -g "markdownlint-cli2@${MARKDOWNLINT_CLI2_VERSION}" + uv tool install "mbake==${MBAKE_VERSION}" + + - name: Check Docker runtime + run: docker info + + - name: Run template tests with act validation + env: + ACT_GITHUB_TOKEN: ${{ github.token }} + run: make test WITH_ACT=1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..59df0fe --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,43 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +env: + MARKDOWNLINT_CLI2_VERSION: 0.22.1 + MBAKE_VERSION: 1.4.6 + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: '3.13' + + - name: Install uv + uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 + + - name: Set up Rust + uses: leynos/shared-actions/.github/actions/setup-rust@455d9ed03477c0026da96c2541ca26569a74acac + with: + toolchain: stable + + - name: Install template test tools + run: | + npm install -g "markdownlint-cli2@${MARKDOWNLINT_CLI2_VERSION}" + uv tool install "mbake==${MBAKE_VERSION}" + + - name: Run template tests + run: make test diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..3355b8c --- /dev/null +++ b/Makefile @@ -0,0 +1,27 @@ +.PHONY: help check-fmt lint test typecheck + +MAKEFLAGS += --no-print-directory + +UV := $(shell command -v uvx 2>/dev/null) +WITH_ACT ?= 0 +ACT_TEST_ENV = $(if $(filter 1 true yes on,$(WITH_ACT)),RUN_ACT_VALIDATION=1,) + +ifeq ($(strip $(UV)),) +$(error uvx is required to run template tests. Install uv from https://docs.astral.sh/uv/getting-started/installation/) +endif + +test: ## Run template tests + $(ACT_TEST_ENV) $(UV) --with pytest-copier --with pyyaml --with syrupy --with make-parser pytest tests/ + +check-fmt: ## Verify template test formatting + $(UV) ruff format --check tests/ + +lint: ## Run template test lint checks + $(UV) ruff check tests/ + +typecheck: ## Run template test type checks + $(UV) --with pytest --with pytest-copier --with pyyaml --with syrupy --with make-parser ty check tests/ + +help: ## Show available targets + @grep -E '^[a-zA-Z_-]+:.*?##' $(MAKEFILE_LIST) | \ + awk 'BEGIN {FS=":.*?## "; printf "Available targets:\n"} {printf " %-15s %s\n", $$1, $$2}' diff --git a/README.md b/README.md index 67eed41..f4db656 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # Generic Copier Template -This repository provides a [Copier](https://copier.readthedocs.io/) template for a Python package. +This repository provides a [Copier](https://copier.readthedocs.io/) template +for a Python package. It offers two flavours: 1. **Python Only** – a pure Python implementation. @@ -14,22 +15,33 @@ The test suite relies on the `pytest-copier` plugin and renders generated projects that run Ruff, Pylint via a PyPy-backed runner, `ty`, pytest, and, when the Rust extension is enabled, Clippy, Whitaker, and nextest-aware Rust tests. -Install the `pytest-copier` test dependency before running this repository's -`pytest` suite. Generated projects install and run their own tooling, including -Ruff, Pylint via PyPy, `ty`, pytest, and, when Rust is enabled, Clippy, Whitaker, -and nextest. +Run the parent template tests through the repository `Makefile`. Run +`make help` to list the available parent Makefile targets. Parent gates include +`make check-fmt`, `make lint`, `make typecheck`, and `make test`. The `test` +target uses `uvx` to provide `pytest-copier`, `pyyaml`, `syrupy`, and +`make-parser` without a manually managed virtual environment: ```bash -pip install pytest-copier +make test ``` -You can also run `scripts/setup_test_deps.sh` to install them automatically. +Run `make test WITH_ACT=1` to include the act-backed workflow integration tests +when `act` and Docker are available. Parent repository CI runs this mode in a +separate act-validation workflow so rendered in-template GitHub workflows do not +hold up the normal template test workflow. + +Generated projects install and run their own tooling, including Ruff, Pylint via +PyPy, `ty`, pytest, and, when Rust is enabled, Clippy, Whitaker, and nextest. + +You can also run `scripts/setup_test_deps.sh` to install parent test +dependencies into the current Python environment manually. ## Generated Quality Gate Flow Figure: The generated `make all` quality gate runs build, formatting, linting, typechecking, and testing. Rust-specific lint and test branches run only when -the Rust extension is enabled. +the Rust extension is enabled. The `audit` target checks Python dependencies +via `pip-audit` and, when Rust is enabled, Rust dependencies via `cargo audit`. ```mermaid flowchart LR @@ -38,17 +50,20 @@ flowchart LR All --> CheckFmt[check-fmt] All --> Lint[lint] All --> Typecheck[typecheck] + All --> Audit[audit] All --> Test[test] Lint --> RuffCheck[ruff check] Lint --> PylintPyPy[pylint-pypy via PyPy] Typecheck --> TyCheck[ty check] + Audit --> PipAudit[pip-audit] Test --> Pytest[pytest -v -n auto] subgraph Rust_extension_enabled[use_rust == true] Lint --> RustDoc[cargo doc] Lint --> Clippy[cargo clippy] Lint --> Whitaker[whitaker --all] + Audit --> CargoAudit[cargo audit] Test --> Nextest[cargo nextest run] Test --> RustDocTests[cargo test --doc] end diff --git a/docs/developers-guide.md b/docs/developers-guide.md index 07908a0..1c91f98 100644 --- a/docs/developers-guide.md +++ b/docs/developers-guide.md @@ -4,10 +4,49 @@ This repository is a Copier template. Source files under `template/` are rendered into generated projects, while tests under `tests/` render those projects and run their public quality gates. +## Parent Repository Makefile + +The root `Makefile` provides developer workflow targets for working on the +template itself. + +- `make help` — lists all `##`-annotated targets. +- `make check-fmt` — runs Ruff formatting checks against the parent template + test suite. +- `make lint` — runs Ruff lint checks against the parent template test suite. +- `make typecheck` — runs `ty check` against the parent template test suite, + supplying the test dependencies through `uvx`. +- `make test` — runs the template test suite via `uvx`, supplying + `pytest-copier`, `pyyaml`, `syrupy`, and `make-parser` without a manually + managed virtual environment. +- `make test WITH_ACT=1` — sets `RUN_ACT_VALIDATION=1` inside the pytest + invocation, enabling the act-backed integration tests that run generated CI + workflows locally. Requires `act` and Docker to be available. + +`uvx` is required; the Makefile aborts with an error if it is not found on +`PATH`. + +## Parent CI Workflows + +The parent repository uses two separate GitHub Actions workflows to keep +Docker-dependent tests isolated from the standard template test gate: + +- `.github/workflows/ci.yml` runs `make test` on every push to `main` and on + all pull requests. It installs `markdownlint-cli2` and `mbake` at pinned + versions. +- `.github/workflows/act-validation.yml` runs `make test WITH_ACT=1`. It + additionally downloads the `act` binary at a pinned `ACT_VERSION`, verifies + its SHA-256 checksum before extraction, and confirms Docker availability via + `docker info`. The workflow exposes `ACT_GITHUB_TOKEN: ${{ github.token }}` + only to the nested act test step so actions requiring `github.token` behave + like they do on GitHub-hosted runners. + ## Makefile Template `template/Makefile.jinja` defines the generated developer workflow. The default `all` target runs build, formatting, linting, typechecking, and tests. +The generated `audit` target is an explicit security gate run by CI. It runs +`pip-audit` for every rendered project and, when `use_rust` is enabled, also +runs `cargo audit` in the Rust extension crate. The generated lint targets are split by language: @@ -15,6 +54,8 @@ The generated lint targets are split by language: - `lint-rust` exists only when `use_rust` is enabled and runs rustdoc, Clippy, and Whitaker. - `lint` delegates to the applicable language-specific targets. +- `audit` exists for both generated variants and runs `pip-audit`; Rust-enabled + variants delegate to `rust-audit` for `cargo audit`. Tool revisions are exposed as Makefile variables such as `PYLINT_PYPY_SHIM_REF` and `WHITAKER_INSTALLER_REV`, so generated projects can @@ -24,9 +65,18 @@ override pins without editing target recipes. `template/.github/workflows/ci.yml.jinja` mirrors the generated local gates. It sets up Python, optionally sets up Rust, runs `make check-fmt`, `make lint`, -and `make typecheck`, then delegates coverage to +`make typecheck`, and `make audit`, then delegates coverage to `leynos/shared-actions/.github/actions/generate-coverage`. +The shared coverage action runs Python coverage through xdist-backed SlipCover +support. Generated pytest discovery is therefore constrained to the top-level +`tests/` tree via `tool.pytest.ini_options.testpaths`; do not add pytest unit +tests under package module directories or `unittests/` subdirectories. + +`template/.github/workflows/act-validation.yml.jinja` keeps rendered workflow +validation separate from generated application CI. It installs `act`, verifies +Docker availability, and runs `make test WITH_ACT=1`. + Rust-enabled workflows pass `rust_extension/Cargo.toml` to the coverage action because the generated Python project root does not contain a Rust manifest. @@ -50,3 +100,45 @@ Rust documentation output, maturin configuration, and cargo error messaging. The optional `act` tests run generated GitHub Actions workflows as black-box integration checks. Their parser separates structured-log handling from the domain assertions so workflow format details stay local to the adapter helpers. + +### Test Helper Modules + +Test helpers are organized under `tests/helpers/` into three modules, each with +a distinct responsibility: + +**`tests/helpers/rendering.py`** — project rendering and command execution. + +- `render_project` wraps the `pytest-copier` fixture to render a temporary + project with explicit template answers. +- `run_quality_gates` runs the generated `make all` target via the rendered + project wrapper. +- `check_generated_import` imports the generated package through `uv run` and + asserts its `hello()` return value. +- `read_generated_file` reads a file from the rendered project root as UTF-8 + text, converting OS errors into `pytest.fail` exceptions. + +**`tests/helpers/generated_files.py`** — file parsing with assertion-focused +error context. + +- `read_generated_text` reads a `Path` and converts `OSError` into + `pytest.fail`. +- `parse_toml_file` reads and parses a TOML file, converting decode errors + into `pytest.fail`. +- `parse_yaml_mapping` parses a YAML string and asserts the result is a + mapping. +- `require_mapping` / `require_sequence` extract nested keys from a parsed + mapping, failing with a schema-path message when absent or the wrong type. + +**`tests/helpers/tooling_contracts.py`** — generated tooling contract +assertions. + +- `assert_common_make_targets` validates Makefile targets shared by all + generated variants. +- `assert_generated_tooling_contracts` is the single entry-point that + validates packaging configuration, AGENTS.md guidance, Makefile wiring, CI + and release workflow structure, and wheel workflow/action contracts. +- `assert_ci_coverage_action_contract` validates the shared coverage action + step and its inputs, including optional Rust cargo-manifest inputs. + +All public helpers carry NumPy-style docstrings. Internal helpers (prefixed +`_`) are private to the module and not part of the test API. diff --git a/scripts/setup_test_deps.sh b/scripts/setup_test_deps.sh index eef3164..ba77007 100755 --- a/scripts/setup_test_deps.sh +++ b/scripts/setup_test_deps.sh @@ -3,4 +3,4 @@ # Generated projects install their own linting and type-checking dependencies. set -euo pipefail -pip install pytest-copier syrupy +pip install pytest-copier pyyaml syrupy make-parser diff --git a/template/.github/actions/build-wheels/action.yml b/template/.github/actions/build-wheels/action.yml index 85680ed..1b0fc51 100644 --- a/template/.github/actions/build-wheels/action.yml +++ b/template/.github/actions/build-wheels/action.yml @@ -16,6 +16,7 @@ runs: - uses: actions/checkout@v4 with: fetch-depth: 0 + persist-credentials: false - name: Set up Python uses: actions/setup-python@v5 with: diff --git a/template/.github/actions/pure-python-wheel/action.yml b/template/.github/actions/pure-python-wheel/action.yml index 656e791..2eeedd9 100644 --- a/template/.github/actions/pure-python-wheel/action.yml +++ b/template/.github/actions/pure-python-wheel/action.yml @@ -17,6 +17,7 @@ runs: - uses: actions/checkout@v4 with: fetch-depth: 0 + persist-credentials: false - name: Set up Python uses: actions/setup-python@v5 with: diff --git a/template/.github/workflows/act-validation.yml.jinja b/template/.github/workflows/act-validation.yml.jinja new file mode 100644 index 0000000..c76ea3c --- /dev/null +++ b/template/.github/workflows/act-validation.yml.jinja @@ -0,0 +1,57 @@ +name: Act Validation + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +jobs: + act-validation: + runs-on: ubuntu-latest + env: + ACT_VERSION: v0.2.84 + MARKDOWNLINT_CLI2_VERSION: 0.22.1 + MBAKE_VERSION: 1.4.6 + steps: + - name: Check out repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: '3.13' + + - name: Install uv + uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 + + {% if use_rust %} + - name: Set up Rust + uses: leynos/shared-actions/.github/actions/setup-rust@455d9ed03477c0026da96c2541ca26569a74acac + with: + toolchain: stable + + {% endif %} + - name: Install act + run: | + act_archive=act_Linux_x86_64.tar.gz + act_url="https://github.com/nektos/act/releases/download/${ACT_VERSION}/${act_archive}" + curl -fsSLo "${act_archive}" "${act_url}" + act_sha256="$( + curl -fsSL "https://api.github.com/repos/nektos/act/releases/tags/${ACT_VERSION}" | + python -c 'import json, sys; release = json.load(sys.stdin); print(next(asset["digest"].split(":", 1)[1] for asset in release["assets"] if asset["name"] == "act_Linux_x86_64.tar.gz"))' + )" + printf '%s %s\n' "${act_sha256}" "${act_archive}" | sha256sum -c - + sudo tar -xzf "${act_archive}" -C /usr/local/bin act + npm install -g "markdownlint-cli2@${MARKDOWNLINT_CLI2_VERSION}" + uv tool install "mbake==${MBAKE_VERSION}" + + - name: Check Docker runtime + run: docker info + + - name: Run tests with act validation + run: make test WITH_ACT=1 diff --git a/template/.github/workflows/build-wheels.yml b/template/.github/workflows/build-wheels.yml index c83b9df..822fd13 100644 --- a/template/.github/workflows/build-wheels.yml +++ b/template/.github/workflows/build-wheels.yml @@ -34,6 +34,8 @@ jobs: cibw_arch: arm64 steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: ./.github/actions/build-wheels with: python-version: ${{ inputs['python-version'] }} diff --git a/template/.github/workflows/ci.yml.jinja b/template/.github/workflows/ci.yml.jinja index dbb218a..c9ec18a 100644 --- a/template/.github/workflows/ci.yml.jinja +++ b/template/.github/workflows/ci.yml.jinja @@ -19,6 +19,8 @@ jobs: steps: - name: Check out repository uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + with: + persist-credentials: false - name: Set up Python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 @@ -30,7 +32,7 @@ jobs: {% if use_rust %} - name: Set up Rust - uses: leynos/shared-actions/.github/actions/setup-rust@d400b079fb6a8fa92f7e7b6c57f3d1c92a4b2d54 + uses: leynos/shared-actions/.github/actions/setup-rust@455d9ed03477c0026da96c2541ca26569a74acac with: toolchain: stable @@ -39,17 +41,21 @@ jobs: with: cache-directories: | ~/.cargo/bin/cargo-nextest + ~/.cargo/bin/cargo-audit ~/.cargo/bin/whitaker-installer ~/.local/bin/whitaker ~/.local/share/whitaker - name: Install Rust lint and test tools + env: + RUSTFLAGS: "" run: | cargo install --locked \ --git https://github.com/leynos/whitaker \ --rev "${WHITAKER_INSTALLER_REV}" \ whitaker-installer whitaker-installer --cranelift + cargo install --locked cargo-audit {% endif %} - name: Install CLI tools @@ -72,8 +78,11 @@ jobs: - name: Run typechecker run: make typecheck + - name: Audit dependencies + run: make audit + - name: Test and Measure Coverage - uses: leynos/shared-actions/.github/actions/generate-coverage@d400b079fb6a8fa92f7e7b6c57f3d1c92a4b2d54 + uses: leynos/shared-actions/.github/actions/generate-coverage@455d9ed03477c0026da96c2541ca26569a74acac with: output-path: coverage.xml format: cobertura diff --git a/template/.github/workflows/release.yml b/template/.github/workflows/release.yml.jinja similarity index 66% rename from template/.github/workflows/release.yml rename to template/.github/workflows/release.yml.jinja index 0cb837d..5d10ea1 100644 --- a/template/.github/workflows/release.yml +++ b/template/.github/workflows/release.yml.jinja @@ -6,7 +6,7 @@ on: - 'v*.*.*' concurrency: - group: release-${{ github.ref }} + group: release-${{ "{{" }} github.ref {{ "}}" }} cancel-in-progress: true permissions: @@ -15,6 +15,12 @@ permissions: jobs: +{% if use_rust %} + build-wheels: + uses: ./.github/workflows/build-wheels.yml + with: + python-version: '3.13' +{% else %} pure-wheel: runs-on: ubuntu-latest steps: @@ -22,21 +28,30 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 + persist-credentials: false - uses: ./.github/actions/pure-python-wheel with: python-version: '3.13' artifact-name: wheels-pure +{% endif %} release: +{% if not use_rust %} # This project has no C or Rust extensions, so cross-platform # builds are unnecessary. Only the pure Python wheel is published. +{% endif %} needs: +{% if use_rust %} + - build-wheels +{% else %} - pure-wheel +{% endif %} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: fetch-depth: 0 + persist-credentials: false - uses: softprops/action-gh-release@v2 with: generate_release_notes: true @@ -47,6 +62,6 @@ jobs: run: | set -eu find dist/wheels-* -type f -name "*.whl" -print0 | \ - xargs -0 -r gh release upload "${{ github.ref_name }}" + xargs -0 -r gh release upload "${{ "{{" }} github.ref_name {{ "}}" }}" --clobber env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ "{{" }} secrets.GITHUB_TOKEN {{ "}}" }} diff --git a/template/.rules/python-00.md b/template/.rules/python-00.md index 5e3d3ce..6ff106d 100644 --- a/template/.rules/python-00.md +++ b/template/.rules/python-00.md @@ -75,14 +75,14 @@ def scale(values: list[float], factor: float) -> list[float]: ## Testing with pytest -- **Colocate unit tests with code** using an `unittests` subdirectory and a - `test_` prefix. This keeps logic and its tests together: +- **Keep tests in the top-level `tests/` tree.** Do not place pytest unit tests + in package module directories or `unittests/` subdirectories. Coverage runs + through xdist-backed SlipCover support, which currently requires tests to be + outside the import package tree. ```text -user_auth/ - models.py - login_flow.py - unittests/ +tests/ + user_auth/ test_models.py test_login_flow.py ``` diff --git a/template/AGENTS.md.jinja b/template/AGENTS.md.jinja index a30823e..bb6c5e4 100644 --- a/template/AGENTS.md.jinja +++ b/template/AGENTS.md.jinja @@ -69,16 +69,33 @@ - Formatting is correct and validated. - **For Python files:** - **Testing:** Passes all relevant unit and behavioural tests (`make test`). + Use `make test WITH_ACT=1` when local GitHub Actions workflow validation + should run through `act`; this sets `RUN_ACT_VALIDATION=1` for the pytest + invocation. - **Linting:** Passes lint checks (`make lint`). - **Formatting:** Adheres to formatting standards (`make check-fmt`; use `make fmt` to apply fixes). - **Typechecking:** Passes type checking (`make typecheck`). + - **Auditing:** Passes dependency vulnerability checks (`make audit`). + - The generated Makefile wiring for these targets is: + - `make check-fmt` runs Ruff formatting checks with + `ruff format --check $(PYTHON_TARGETS)`. + - `make lint` runs `make lint-python`; `make lint-python` runs + `ruff check $(PYTHON_TARGETS)` and the PyPy-backed Pylint runner against + `$(PYLINT_TARGETS)`. + - `make typecheck` runs `ty check $(PYTHON_TARGETS)`. + - `make test` runs `pytest -v -n $(PYTEST_XDIST_WORKERS)` and honours + `WITH_ACT=1` through `RUN_ACT_VALIDATION=1`. + - `make audit` runs `pip-audit`. + - Temporary lint suppressions must include a link to the planned fix or + implementation that will remove the suppression. {% if use_rust -%} - For Rust files: - **Testing:** Passes relevant unit and behavioural tests (`make test`). - **Linting:** Passes lint checks (`make lint`). - **Formatting:** Adheres to formatting standards (`make check-fmt`; use `make fmt` to apply fixes). + - **Auditing:** Passes Rust dependency vulnerability checks (`make audit`). {% endif %} - **Markdown files (`.md` only):** - **Linting:** Passes markdown lint checks (`make markdownlint`). @@ -127,8 +144,20 @@ - For Python work, use `pytest` for unit tests and `pytest-bdd` for behavioural tests. Cover happy paths, unhappy paths, and relevant edge cases. +- Keep pytest tests in the top-level `tests/` tree. Do not place unit tests in + package module directories or `unittests/` subdirectories, because CI coverage + uses xdist-backed SlipCover support. - Snapshot tests (using `syrupy`) should be provided where multivariant output - format consistency is relevant to the requirements. + format consistency is relevant to the requirements. Snapshot tests must + capture meaningful, reviewer-useful contracts rather than generic dumps of + broad objects. Keep snapshots focused on stable output boundaries, pair them + with semantic assertions for the behaviours they represent, and avoid + snapshot-only coverage for logic that can be asserted directly. Redact or + normalize nondeterministic fields such as timestamps, absolute paths, random + identifiers, ordering-dependent maps, hostnames, and environment-specific + values before snapshotting. Do not accept brittle snapshots that churn after + harmless formatting or dependency changes; narrow the captured output until a + snapshot failure identifies a real contract change. - Add end-to-end tests where a change affects externally observable workflows, integration contracts, persistence, command-line behaviour, network boundaries, user interface flows, or other system-level behaviour. @@ -151,27 +180,41 @@ project: - `make check-fmt` executes: ```sh - cargo fmt --workspace -- --check + cargo fmt --manifest-path rust_extension/Cargo.toml --all -- --check ``` - validating formatting across the entire workspace without modifying files. + validating formatting for the Rust extension without modifying files. - `make lint` executes: ```sh - cargo clippy --workspace --all-targets --all-features -- -D warnings + cargo doc --no-deps --manifest-path rust_extension/Cargo.toml + cargo clippy --manifest-path rust_extension/Cargo.toml --all-targets --all-features -- -D warnings + whitaker --all -- --all-targets --all-features ``` - linting every target with all features enabled and denying all Clippy - warnings. + documenting and linting every Rust extension target with all features + enabled and warnings denied. - `make test` executes: ```sh - cargo test --workspace + cargo nextest run --manifest-path rust_extension/Cargo.toml --all-targets --all-features + # or, when cargo-nextest is unavailable: + cargo test --manifest-path rust_extension/Cargo.toml --all-targets --all-features + cargo test --doc --manifest-path rust_extension/Cargo.toml --all-features ``` - running the full workspace test suite. Use `make fmt` - (`cargo fmt --workspace`) to apply formatting fixes reported by the - formatter check. + running the Rust extension test suite and documentation tests. Use + `make fmt` (`cargo fmt --manifest-path rust_extension/Cargo.toml --all`) to + apply formatting fixes reported by the formatter check. + - `make audit` executes: + + ```sh + pip-audit + cargo audit + ``` + + checking Python dependencies and Rust extension dependencies for known + vulnerabilities. - Clippy warnings MUST be disallowed. - Fix any warnings emitted during tests in code instead of silencing them. - Where a function is too long, extract meaningfully named helper functions @@ -183,8 +226,9 @@ project: - Ensure that new features are validated with unit and behavioural tests before release using `rstest` and `rstest-bdd`. Cover happy paths, unhappy paths, and relevant edge cases. -- For Rust: snapshot tests (using `insta`) should be provided where multivariant - output format consistency is relevant to the requirements. +- For Rust: snapshot tests (using `insta`) should follow the same snapshot + quality criteria and should be provided where multivariant output format + consistency is relevant to the requirements. - Add end-to-end tests where a change affects externally observable workflows, integration contracts, persistence, command-line behaviour, network boundaries, user interface flows, or other system-level behaviour. diff --git a/template/Makefile.jinja b/template/Makefile.jinja index 576e78d..3882c4e 100644 --- a/template/Makefile.jinja +++ b/template/Makefile.jinja @@ -9,6 +9,8 @@ USER_BIN_PATH := $(HOME)/.cargo/bin:$(HOME)/.local/bin:$(HOME)/.bun/bin TOOLS = $(MDFORMAT_ALL) $(MDLINT) VENV_TOOLS = pytest UV_ENV = PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1 UV_CACHE_DIR=.uv-cache UV_TOOL_DIR=.uv-tools +WITH_ACT ?= 0 +ACT_TEST_ENV = $(if $(filter 1 true yes on,$(WITH_ACT)),RUN_ACT_VALIDATION=1,) PYTEST_XDIST_WORKERS ?= auto PYTHON_TARGETS ?= {{ package_name }} tests PYLINT_PYTHON ?= pypy @@ -34,8 +36,8 @@ WHITAKER_INSTALLER_REV ?= f768c2e53c47df13658af1168a67851d388750bf WHITAKER ?= $(or $(shell command -v whitaker 2>/dev/null),$(wildcard $(USER_WHITAKER)),whitaker) {% endif %} -.PHONY: help all clean build build-release lint lint-python fmt check-fmt \ - markdownlint nixie test typecheck $(TOOLS) $(VENV_TOOLS){% if use_rust %} lint-rust whitaker{% endif %} +.PHONY: help all audit clean build build-release lint lint-python fmt check-fmt \ + markdownlint nixie test typecheck $(TOOLS) $(VENV_TOOLS){% if use_rust %} lint-rust rust-audit whitaker{% endif %} .DEFAULT_GOAL := all @@ -148,6 +150,16 @@ typecheck: build ## Run typechecking $(UV_ENV) $(UV) run ty --version $(UV_ENV) $(UV) run ty check $(PYTHON_TARGETS) +audit: build ## Audit dependencies for known vulnerabilities + $(UV_ENV) $(UV) run pip-audit +{% if use_rust %} + $(MAKE) rust-audit + +rust-audit: ## Audit Rust extension dependencies for known vulnerabilities + $(call ensure_cargo) + cd $(RUST_CRATE_DIR) && $(CARGO) audit +{% endif %} + markdownlint: $(MDLINT) ## Lint Markdown files env -u NO_COLOR $(MDLINT) '**/*.md' @@ -156,16 +168,14 @@ nixie: ## Validate Mermaid diagrams $(NIXIE) --no-sandbox test: build $(VENV_TOOLS) ## Run tests - $(UV_ENV) $(UV) run pytest -v -n $(PYTEST_XDIST_WORKERS) + $(UV_ENV) $(ACT_TEST_ENV) $(UV) run pytest -v -n $(PYTEST_XDIST_WORKERS) {% if use_rust %} @test -n "$(CARGO_AVAILABLE)" || { \ printf "Error: cargo is required for Rust tests, but '%s' was not found on PATH\n" "$(CARGO)" >&2; \ exit 1; \ } RUSTFLAGS="$(RUST_FLAGS)" $(CARGO) $(TEST_CMD) $(TEST_FLAGS) $(BUILD_JOBS) -ifneq ($(TEST_CMD),test) RUSTFLAGS="$(RUST_FLAGS)" $(CARGO) test --doc --manifest-path $(RUST_CRATE_DIR)/Cargo.toml --all-features -endif {% endif %} help: ## Show available targets diff --git a/template/docs/developers-guide.md b/template/docs/developers-guide.md index 3b84a63..5723a94 100644 --- a/template/docs/developers-guide.md +++ b/template/docs/developers-guide.md @@ -9,6 +9,10 @@ The public entrypoint for formatting, linting, typechecking, and tests is failure, and changes should be reconciled with the aggregate gate before being considered complete. +Run `make audit` as the dependency vulnerability gate. It runs `pip-audit` for +Python dependencies, and Rust-enabled projects also run `cargo audit` from the +`rust_extension` crate directory. + ## Automation scripts The [Scripting standards](scripting-standards.md) document provides guidance for @@ -28,10 +32,13 @@ actions under `.github/`. - `.github/workflows/ci.yml` runs on pushes to `main` and on pull requests. It sets up Python 3.13, installs `uv`, validates the generated `Makefile` with - `mbake`, runs `make build`, `make check-fmt`, `make lint`, and - `make typecheck`, then delegates coverage generation to the shared coverage + `mbake`, runs `make build`, `make check-fmt`, `make lint`, `make typecheck`, + and `make audit`, then delegates coverage generation to the shared coverage action. When the Rust extension is enabled, it also sets up Rust, installs Rust lint and test tools, and passes `rust_extension/Cargo.toml` to coverage. +- `.github/workflows/act-validation.yml` runs rendered workflow validation in a + separate workflow. It installs `act`, checks Docker availability, and runs + `make test WITH_ACT=1` outside the coverage path. - `.github/workflows/release.yml` publishes wheels when a `v*.*.*` tag is pushed. It builds a pure Python wheel, creates a GitHub release with generated release notes, downloads wheel artifacts, and uploads them to the tag release. diff --git a/template/docs/users-guide.md.jinja b/template/docs/users-guide.md.jinja index 9070e0c..2b9e639 100644 --- a/template/docs/users-guide.md.jinja +++ b/template/docs/users-guide.md.jinja @@ -12,11 +12,17 @@ these targets in order: - `lint`: run `lint-python` and, when Rust is enabled, `lint-rust`. - `typecheck`: run `ty check`. - `test`: run pytest and, when Rust is enabled, Rust tests. +- `audit`: run `pip-audit` and, when Rust is enabled, `cargo audit`. The `lint-python` target runs Ruff followed by Pylint via a PyPy-backed runner. The Pylint runner is installed through `uv tool run` from the pinned `pylint-pypy-shim` repository. +Pytest discovery is limited to the top-level `tests/` tree. Keep generated +project unit tests there rather than in package module directories or +`unittests/` subdirectories, because CI coverage runs through xdist-backed +SlipCover support. + When the Rust extension is enabled, `lint-rust` runs: - `cargo doc` with warnings denied; @@ -26,6 +32,13 @@ When the Rust extension is enabled, `lint-rust` runs: The generated Makefile installs Whitaker on demand before local Rust linting when it is not already available. +## Dependency Auditing + +Run `make audit` to check generated project dependencies for known +vulnerabilities. All generated projects run `pip-audit` against the Python +environment created by `uv sync --group dev`. Rust-enabled projects also run +`cargo audit` from the `rust_extension` crate directory. + ## Rust Test Behaviour Rust-enabled projects use `cargo nextest run` when `cargo-nextest` is available. @@ -36,6 +49,21 @@ If cargo is missing from the local environment, generated Rust test targets fail early with a clear error instead of falling through to an unusable `cargo` invocation. +## Local GitHub Actions Validation + +The generated Makefile supports optional local workflow validation using +[`act`](https://github.com/nektos/act). When `act` is installed and Docker is +available, pass `WITH_ACT=1` to the `test` target: + +```bash +make test WITH_ACT=1 +``` + +This sets `RUN_ACT_VALIDATION=1` for the pytest invocation, enabling the +act-based integration tests that run the generated CI workflow locally. +Omitting `WITH_ACT` (or setting it to `0`) skips act validation; the rest of +the test suite runs unchanged. + ## Cleaning Local State Run `make clean` to remove local build and cache outputs, including `.venv`, diff --git a/template/pyproject.toml.jinja b/template/pyproject.toml.jinja index c2a8c20..6689f87 100644 --- a/template/pyproject.toml.jinja +++ b/template/pyproject.toml.jinja @@ -10,6 +10,7 @@ dependencies = [] [dependency-groups] dev = [ "pytest", + "pip-audit", "ruff", "pyright", "ty", @@ -295,6 +296,7 @@ enable = [ [tool.pytest.ini_options] # Tests automatically killed after seconds elapsed timeout = 30 +testpaths = ["tests"] [tool.uv] package = true diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..3b56526 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,25 @@ +"""Support utilities for the Copier template validation test suite. + +The package exposes shared helpers and fixtures used by the repository tests, +including rendered-file parsers, generated-project command helpers, workflow +contract assertions, container runtime environment helpers, and pytest fixtures +loaded from :mod:`tests.conftest`. + +Import the package directly when a test only needs package discovery, or import +specific utilities from their modules when using them in assertions. Importing +``tests`` has no side effects beyond normal package initialisation; runtime +probing for tools such as Docker, Podman, or ``act`` stays inside fixtures and +helper functions. + +Examples +-------- +Use helpers from the package in template tests:: + + import tests + from tests.helpers.generated_files import parse_yaml_mapping + from tests.utilities import docker_environment + + assert tests.__doc__ + workflow = parse_yaml_mapping("name: CI\n", "workflow") + env = docker_environment() +""" diff --git a/tests/__snapshots__/test_template.ambr b/tests/__snapshots__/test_template.ambr index 3c4f4c2..eee44c2 100644 --- a/tests/__snapshots__/test_template.ambr +++ b/tests/__snapshots__/test_template.ambr @@ -53,9 +53,11 @@ lint Run linters lint-python Run Python linters typecheck Run typechecking + audit Audit dependencies for known vulnerabilities markdownlint Lint Markdown files nixie Validate Mermaid diagrams test Run tests help Show available targets + ''' # --- diff --git a/tests/conftest.py b/tests/conftest.py index 4f21e9b..8a17760 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -24,6 +24,7 @@ from __future__ import annotations import os +import re import shutil import subprocess @@ -31,6 +32,9 @@ from tests.utilities import docker_environment +MINIMUM_ACT_VERSION = (0, 2, 84) +ACT_VERSION_PATTERN = re.compile(r"\bact version v?(\d+)\.(\d+)\.(\d+)\b") + @pytest.fixture(scope="session") def copier_template_paths() -> list[str]: @@ -110,6 +114,36 @@ def _runtime_info_commands() -> list[list[str]]: return commands +def _parse_act_version(output: str) -> tuple[int, int, int] | None: + """Return the semantic act version from command output.""" + match = ACT_VERSION_PATTERN.search(output) + if match is None: + return None + major, minor, patch = match.groups() + return int(major), int(minor), int(patch) + + +def _installed_act_version() -> tuple[int, int, int]: + """Return the installed act version or skip when it cannot be parsed.""" + try: + version = subprocess.run( + ["act", "--version"], + text=True, + capture_output=True, + check=False, + timeout=30, + ) + except (OSError, subprocess.SubprocessError) as exc: + pytest.skip(f"could not determine act version: {exc}") + parsed_version = _parse_act_version(version.stdout) + if parsed_version is None: + pytest.skip( + "could not parse act version from 'act --version' output: " + f"{version.stdout.strip() or version.stderr.strip()}" + ) + return parsed_version + + @pytest.fixture def act_ready() -> None: """Skip act-backed tests unless local workflow validation can run. @@ -145,6 +179,14 @@ def test_workflow(act_ready: None) -> None: pytest.skip("set RUN_ACT_VALIDATION=1 to run act workflow validation") if shutil.which("act") is None: pytest.skip("act is not installed") + act_version = _installed_act_version() + if act_version < MINIMUM_ACT_VERSION: + found_version = ".".join(str(part) for part in act_version) + minimum_version = ".".join(str(part) for part in MINIMUM_ACT_VERSION) + pytest.skip( + f"act >= {minimum_version} is required for Node24 action runtime " + f"support; found act version {found_version}" + ) info_commands = _runtime_info_commands() if not info_commands: pytest.skip("docker-compatible container runtime is not installed") diff --git a/tests/helpers/__init__.py b/tests/helpers/__init__.py new file mode 100644 index 0000000..2332239 --- /dev/null +++ b/tests/helpers/__init__.py @@ -0,0 +1,27 @@ +"""Shared test utilities for rendered Copier template tests. + +The helpers package groups reusable support code for tests that render Copier +projects and inspect their generated files. It contains modules for generated +file parsing, rendered-project command execution, and tooling or workflow +contract assertions. Fixtures remain in :mod:`tests.conftest`, while runtime +container helpers remain in :mod:`tests.utilities`. + +Import helper functions from the module that owns their behaviour. Test modules +may also import common helpers from ``tests.helpers`` when this package re-exports +them, but module-level imports keep ownership clearer for larger assertions. + +Examples +-------- +Use the generated-file and contract helpers in a template test:: + + from tests.helpers.generated_files import parse_yaml_mapping + from tests.helpers.tooling_contracts import assert_ci_coverage_action_contract + + workflow = parse_yaml_mapping(ci_workflow, "CI workflow") + assert "jobs" in workflow + assert_ci_coverage_action_contract( + ci_workflow=ci_workflow, + package_name="example_pkg", + use_rust=False, + ) +""" diff --git a/tests/helpers/agents_contracts.py b/tests/helpers/agents_contracts.py new file mode 100644 index 0000000..947108b --- /dev/null +++ b/tests/helpers/agents_contracts.py @@ -0,0 +1,190 @@ +"""Validate rendered AGENTS.md contracts against generated Makefiles. + +This module contains assistant-guidance assertions for rendered projects. It +uses ``tests.helpers.makefile_contracts`` to parse generated Makefiles so the +documented targets and command flags are checked against the actual recipes. +""" + +from __future__ import annotations + +import re + +from tests.helpers.makefile_contracts import _parse_makefile_rules + + +def _assert_agents_contracts(agents: str) -> None: + """Assert generated assistant guidance documents act-enabled testing.""" + assert "make test WITH_ACT=1" in agents, ( + "expected generated AGENTS.md to document act-enabled test runs" + ) + assert "RUN_ACT_VALIDATION=1" in agents, ( + "expected generated AGENTS.md to describe the pytest act environment" + ) + assert "meaningful, reviewer-useful contracts" in agents, ( + "expected generated AGENTS.md to require meaningful snapshot contracts" + ) + assert "generic dumps" in agents, ( + "expected generated AGENTS.md to warn against generic snapshots" + ) + assert "semantic assertions" in agents, ( + "expected generated AGENTS.md to pair snapshots with semantic assertions" + ) + assert "normalize nondeterministic fields" in agents, ( + "expected generated AGENTS.md to require nondeterministic field redaction" + ) + assert "brittle snapshots" in agents, ( + "expected generated AGENTS.md to warn against brittle snapshot churn" + ) + assert "Temporary lint suppressions must include a link" in agents, ( + "expected generated AGENTS.md to require linked plans for temporary " + "lint suppressions" + ) + assert "Keep pytest tests in the top-level `tests/` tree" in agents, ( + "expected generated AGENTS.md to keep pytest discovery compatible with " + "xdist-backed coverage" + ) + + +def _assert_agents_make_targets_mirror_makefile( + *, agents: str, makefile: str, package_name: str, use_rust: bool +) -> None: + """Assert AGENTS.md make target references match parsed Makefile commands.""" + documented_targets = _documented_make_targets(agents) + makefile_rules = _parse_makefile_rules(makefile) + makefile_targets = set(makefile_rules) + missing_targets = sorted(documented_targets - makefile_targets) + assert not missing_targets, ( + "expected every make target documented in generated AGENTS.md to exist " + f"in the generated Makefile, missing: {missing_targets}" + ) + required_documented_targets = { + "audit", + "check-fmt", + "fmt", + "lint", + "markdownlint", + "nixie", + "test", + "typecheck", + } + missing_documented_targets = sorted( + required_documented_targets - documented_targets + ) + assert not missing_documented_targets, ( + "expected generated AGENTS.md to document the generated Makefile quality " + f"gate targets, missing: {missing_documented_targets}" + ) + _assert_documented_command_flags( + agents=agents, + makefile_rules=makefile_rules, + package_name=package_name, + use_rust=use_rust, + ) + + +def _assert_documented_command_flags( + *, + agents: str, + makefile_rules: dict[str, list[str]], + package_name: str, + use_rust: bool, +) -> None: + """Assert documented make command flags are present in parsed recipes.""" + python_targets = f"{package_name} tests" + command_contracts = { + "check-fmt": [ + ("AGENTS.md", "ruff format --check $(PYTHON_TARGETS)"), + ("Makefile", f"ruff format --check {python_targets}"), + ], + "lint-python": [ + ("AGENTS.md", "ruff check $(PYTHON_TARGETS)"), + ("Makefile", f"ruff check {python_targets}"), + ], + "typecheck": [ + ("AGENTS.md", "ty check $(PYTHON_TARGETS)"), + ("Makefile", f"ty check {python_targets}"), + ], + "audit": [ + ("AGENTS.md", "pip-audit"), + ("Makefile", "pip-audit"), + ], + "test": [ + ("AGENTS.md", "pytest -v -n $(PYTEST_XDIST_WORKERS)"), + ("Makefile", "pytest -v -n auto"), + ], + } + if use_rust: + command_contracts.update( + { + "check-fmt": [ + *command_contracts["check-fmt"], + ( + "AGENTS.md", + "cargo fmt --manifest-path rust_extension/Cargo.toml", + ), + ("AGENTS.md", "--all -- --check"), + ("Makefile", "fmt --manifest-path rust_extension/Cargo.toml"), + ("Makefile", "--all -- --check"), + ], + "lint-rust": [ + ("AGENTS.md", "cargo doc --no-deps"), + ( + "AGENTS.md", + "cargo clippy --manifest-path rust_extension/Cargo.toml", + ), + ("AGENTS.md", "--all-targets --all-features -- -D warnings"), + ("AGENTS.md", "whitaker --all -- --all-targets --all-features"), + ("Makefile", "doc --no-deps"), + ("Makefile", "clippy --manifest-path rust_extension/Cargo.toml"), + ("Makefile", "--all-targets --all-features -- -D warnings"), + ("Makefile", "--all -- --all-targets --all-features"), + ], + "test": [ + *command_contracts["test"], + ( + "AGENTS.md", + "cargo nextest run --manifest-path rust_extension/Cargo.toml", + ), + ( + "AGENTS.md", + "cargo test --manifest-path rust_extension/Cargo.toml", + ), + ( + "AGENTS.md", + "cargo test --doc --manifest-path rust_extension/Cargo.toml", + ), + ("Makefile", "--manifest-path rust_extension/Cargo.toml"), + ("Makefile", "--all-targets --all-features"), + ( + "Makefile", + "test --doc --manifest-path rust_extension/Cargo.toml", + ), + ], + "audit": [ + *command_contracts["audit"], + ("AGENTS.md", "cargo audit"), + ("Makefile", "$(MAKE) rust-audit"), + ], + "rust-audit": [ + ("Makefile", "cd rust_extension"), + ("Makefile", ") audit"), + ], + } + ) + for target, checks in command_contracts.items(): + assert target in makefile_rules, ( + "expected generated Makefile rules to include target " + f"{target!r} from command_contracts" + ) + commands = "\n".join(makefile_rules[target]) + for source, fragment in checks: + haystack = agents if source == "AGENTS.md" else commands + assert fragment in haystack, ( + f"expected generated {source} {target!r} command contract to " + f"include {fragment!r}" + ) + + +def _documented_make_targets(agents: str) -> set[str]: + """Return make targets referenced in rendered AGENTS.md guidance.""" + return set(re.findall(r"`make ([a-zA-Z][a-zA-Z_-]*)", agents)) diff --git a/tests/helpers/ci_contracts.py b/tests/helpers/ci_contracts.py new file mode 100644 index 0000000..2226dec --- /dev/null +++ b/tests/helpers/ci_contracts.py @@ -0,0 +1,248 @@ +"""Validate rendered CI and act-validation workflow contracts. + +This module owns generated CI workflow assertions, including coverage-action +inputs, act-validation workflow structure, and shared checkout credential +checks. Release workflow helpers import ``_checkout_steps_disable_credentials`` +from here so checkout security behaviour is asserted consistently. +""" + +from __future__ import annotations + +from typing import Any + +from tests.helpers.generated_files import ( + parse_yaml_mapping, + require_mapping, + require_sequence, +) + + +def assert_ci_coverage_action_contract( + *, ci_workflow: str, package_name: str, use_rust: bool +) -> None: + """Assert generated CI coverage inputs used by act validation. + + Parameters + ---------- + ci_workflow : str + UTF-8 text of the generated CI workflow. + package_name : str + Generated Python import package name used to derive the coverage + artefact suffix. + use_rust : bool + Whether the rendered variant includes the optional Rust extension. + + Returns + ------- + None + The helper returns after the coverage action contract matches + expectations. + + Raises + ------ + AssertionError + Raised when checkout credentials, coverage action pinning, coverage + output settings, artefact naming, or Rust manifest inputs are wrong. + + Examples + -------- + Validate a rendered CI workflow's coverage step:: + + assert_ci_coverage_action_contract( + ci_workflow=ci_workflow, + package_name="example_pkg", + use_rust=True, + ) + """ + parsed_ci_workflow = _parse_ci_workflow(ci_workflow) + jobs = require_mapping(parsed_ci_workflow, "jobs", "CI workflow") + lint_test = require_mapping(jobs, "lint-test", "CI workflow jobs") + steps = require_sequence(lint_test, "steps", "CI lint-test job") + assert _checkout_steps_disable_credentials(steps), ( + "expected CI checkout steps to disable credential persistence" + ) + coverage_steps = [ + step + for step in steps + if isinstance(step, dict) and step.get("name") == "Test and Measure Coverage" + ] + assert len(coverage_steps) == 1, "expected one shared coverage action step" + coverage_step = coverage_steps[0] + assert ( + coverage_step.get("uses") + == "leynos/shared-actions/.github/actions/generate-coverage" + "@455d9ed03477c0026da96c2541ca26569a74acac" + ), "expected CI to use the pinned shared coverage action" + coverage_inputs = require_mapping(coverage_step, "with", "coverage step") + assert coverage_inputs.get("output-path") == "coverage.xml", ( + "expected CI coverage output path to match the act assertion" + ) + assert coverage_inputs.get("format") == "cobertura", ( + "expected CI coverage format to match the CodeScene upload" + ) + assert coverage_inputs.get("artefact-name-suffix") == package_name.replace( + "_", "-" + ), "expected package-specific coverage artefact name suffix" + if use_rust: + assert coverage_inputs.get("cargo-manifest") == "rust_extension/Cargo.toml", ( + "expected Rust variant to pass the extension manifest to coverage" + ) + else: + assert "cargo-manifest" not in coverage_inputs, ( + "expected pure-Python variant to omit Rust coverage inputs" + ) + + +def _parse_ci_workflow(ci_workflow: str) -> dict[str, Any]: + """Parse a generated CI workflow as a YAML mapping.""" + return parse_yaml_mapping(ci_workflow, "CI workflow") + + +def _assert_ci_workflow_contracts( + *, parsed_ci_workflow: dict[str, Any], ci_workflow: str, use_rust: bool +) -> None: + """Assert generated CI workflow contracts. + + Parameters + ---------- + parsed_ci_workflow : dict[str, Any] + Parsed generated CI workflow mapping. + ci_workflow : str + UTF-8 text of the generated CI workflow, used for string-level contract + assertions. + use_rust : bool + Whether the rendered variant includes the optional Rust extension. + """ + jobs = require_mapping(parsed_ci_workflow, "jobs", "CI workflow") + lint_test = require_mapping(jobs, "lint-test", "CI workflow jobs") + steps = require_sequence(lint_test, "steps", "CI lint-test job") + assert _checkout_steps_disable_credentials(steps), ( + "expected generated CI workflow to disable checkout credentials" + ) + assert "make check-fmt" in ci_workflow, ( + "expected generated CI workflow to run the formatting gate" + ) + assert "make lint" in ci_workflow, ( + "expected generated CI workflow to run the lint gate" + ) + assert "make typecheck" in ci_workflow, ( + "expected generated CI workflow to run the typecheck gate" + ) + assert "make audit" in ci_workflow, ( + "expected generated CI workflow to run the dependency audit gate" + ) + assert "make test WITH_ACT=1" not in ci_workflow, ( + "expected generated main CI workflow to leave act validation to a " + "separate workflow" + ) + assert "make build" in ci_workflow, ( + "expected generated CI workflow to build the project before checks" + ) + assert "leynos/shared-actions/.github/actions/generate-coverage" in ci_workflow, ( + "expected generated CI workflow to use the shared coverage action" + ) + if use_rust: + assert "leynos/shared-actions/.github/actions/setup-rust" in ci_workflow, ( + "expected Rust variant CI to set up Rust" + ) + rust_tool_steps = [ + step + for step in steps + if isinstance(step, dict) + and step.get("name") == "Install Rust lint and test tools" + ] + assert len(rust_tool_steps) == 1, ( + "expected Rust variant CI to install Rust lint and test tools once" + ) + rust_tool_env = require_mapping( + rust_tool_steps[0], + "env", + "Install Rust lint and test tools step", + ) + assert rust_tool_env.get("RUSTFLAGS") == "", ( + "expected Rust tool installation step to clear inherited RUSTFLAGS" + ) + assert "Cache Rust lint and test tools" in ci_workflow, ( + "expected Rust variant CI to cache Rust tools" + ) + assert "cargo-manifest: rust_extension/Cargo.toml" in ci_workflow, ( + "expected Rust variant CI to pass the Rust manifest to coverage" + ) + assert "cargo install --locked cargo-audit" in ci_workflow, ( + "expected Rust variant CI to install cargo-audit" + ) + else: + assert "setup-rust" not in ci_workflow, ( + "expected pure-Python CI to omit Rust setup" + ) + assert "cargo-manifest" not in ci_workflow, ( + "expected pure-Python CI to omit Rust coverage inputs" + ) + assert "cargo-audit" not in ci_workflow, ( + "expected pure-Python CI to omit Rust audit installation" + ) + + +def _assert_act_validation_workflow_contracts( + *, act_validation_workflow: str, use_rust: bool +) -> None: + """Assert generated act-validation workflow contracts.""" + assert "name: Act Validation" in act_validation_workflow, ( + "expected generated act-validation workflow to be named" + ) + assert "ACT_VERSION: v0.2.84" in act_validation_workflow, ( + "expected generated act-validation workflow to pin act" + ) + assert "permissions:\n contents: read" in act_validation_workflow, ( + "expected generated act-validation workflow to restrict GITHUB_TOKEN" + ) + assert "MARKDOWNLINT_CLI2_VERSION: 0.22.1" in act_validation_workflow, ( + "expected generated act-validation workflow to pin markdownlint-cli2" + ) + assert "MBAKE_VERSION: 1.4.6" in act_validation_workflow, ( + "expected generated act-validation workflow to pin mbake" + ) + assert "act_Linux_x86_64.tar.gz" in act_validation_workflow, ( + "expected generated act-validation workflow to install act" + ) + assert "${ACT_VERSION}/${act_archive}" in act_validation_workflow, ( + "expected generated act-validation workflow to build the act URL from ACT_VERSION" + ) + assert "sha256sum -c -" in act_validation_workflow, ( + "expected generated act-validation workflow to verify the act archive checksum" + ) + assert 'npm install -g "markdownlint-cli2@${MARKDOWNLINT_CLI2_VERSION}"' in ( + act_validation_workflow + ), "expected generated act-validation workflow to install pinned markdownlint-cli2" + assert 'uv tool install "mbake==${MBAKE_VERSION}"' in act_validation_workflow, ( + "expected generated act-validation workflow to install pinned mbake" + ) + assert "docker info" in act_validation_workflow, ( + "expected generated act-validation workflow to verify Docker" + ) + assert "make test WITH_ACT=1" in act_validation_workflow, ( + "expected generated act-validation workflow to run act-enabled tests" + ) + if use_rust: + assert "leynos/shared-actions/.github/actions/setup-rust" in ( + act_validation_workflow + ), "expected Rust variant act-validation workflow to set up Rust" + else: + assert "setup-rust" not in act_validation_workflow, ( + "expected pure-Python act-validation workflow to omit Rust setup" + ) + + +def _checkout_steps_disable_credentials(steps: list[Any]) -> bool: + """Return whether all checkout steps disable credential persistence.""" + checkout_steps = [ + step + for step in steps + if isinstance(step, dict) + and str(step.get("uses", "")).startswith("actions/checkout@") + ] + return bool(checkout_steps) and all( + isinstance(step.get("with"), dict) + and step["with"].get("persist-credentials") is False + for step in checkout_steps + ) diff --git a/tests/helpers/generated_files.py b/tests/helpers/generated_files.py new file mode 100644 index 0000000..18eb585 --- /dev/null +++ b/tests/helpers/generated_files.py @@ -0,0 +1,183 @@ +"""Parse generated project files for template contract tests. + +This module owns assertion-focused file reading and structured data parsing for +rendered Copier projects. Rendering helpers use ``read_generated_text`` so +filesystem errors become pytest failures consistently, while tooling contract +helpers consume ``parse_yaml_mapping``, ``require_mapping``, and +``require_sequence`` to keep workflow schema checks readable. Keep raw +generated-file I/O here so template tests share one error-reporting boundary. +""" + +from __future__ import annotations + +import tomllib +from pathlib import Path +from typing import Any + +import pytest +import yaml + + +def read_generated_text(path: Path) -> str: + """Read a generated file with assertion-focused error context. + + Parameters + ---------- + path + Path to the generated file to read. + + Returns + ------- + str + UTF-8 decoded file contents. + + Raises + ------ + pytest.fail.Exception + Raised when the file cannot be read. + + Examples + -------- + Read a rendered workflow before parsing it:: + + workflow = read_generated_text(project / ".github/workflows/ci.yml") + """ + try: + return path.read_text(encoding="utf-8") + except OSError as error: + pytest.fail(f"could not read generated file {path}: {error}") + + +def parse_toml_file(path: Path) -> dict[str, Any]: + """Parse generated TOML with assertion-focused error context. + + Parameters + ---------- + path + Path to the generated TOML file. + + Returns + ------- + dict[str, Any] + Parsed TOML mapping. + + Raises + ------ + pytest.fail.Exception + Raised when the file cannot be read or the TOML is invalid. + + Examples + -------- + Parse generated project metadata:: + + pyproject = parse_toml_file(project / "pyproject.toml") + """ + text = read_generated_text(path) + try: + parsed = tomllib.loads(text) + except tomllib.TOMLDecodeError as error: + pytest.fail(f"could not parse generated TOML {path}: {error}") + return parsed + + +def parse_yaml_mapping(text: str, label: str) -> dict[str, Any]: + """Parse generated YAML as a mapping with clear failure context. + + Parameters + ---------- + text + YAML document text to parse. + label + Human-readable label used in assertion failure messages. + + Returns + ------- + dict[str, Any] + Parsed YAML mapping. + + Raises + ------ + pytest.fail.Exception + Raised when the YAML is invalid or the parsed document is not a mapping. + + Examples + -------- + Parse a rendered CI workflow:: + + workflow = parse_yaml_mapping(ci_workflow, "CI workflow") + """ + try: + parsed = yaml.safe_load(text) + except yaml.YAMLError as error: + pytest.fail(f"could not parse generated {label}: {error}") + if not isinstance(parsed, dict): + pytest.fail(f"expected generated {label} to parse as a mapping") + return parsed + + +def require_mapping(mapping: dict[str, Any], key: str, label: str) -> dict[str, Any]: + """Return a nested mapping or fail with the missing schema path. + + Parameters + ---------- + mapping + Parent mapping to inspect. + key + Nested key expected to contain a mapping. + label + Human-readable schema path used in assertion failure messages. + + Returns + ------- + dict[str, Any] + Nested mapping value. + + Raises + ------ + pytest.fail.Exception + Raised when the key is missing or the value is not a mapping. + + Examples + -------- + Extract workflow jobs after parsing YAML:: + + jobs = require_mapping(workflow, "jobs", "CI workflow") + """ + value = mapping.get(key) + if not isinstance(value, dict): + pytest.fail(f"expected {label} to include mapping key {key!r}") + return value + + +def require_sequence(mapping: dict[str, Any], key: str, label: str) -> list[Any]: + """Return a nested sequence or fail with the missing schema path. + + Parameters + ---------- + mapping + Parent mapping to inspect. + key + Nested key expected to contain a sequence. + label + Human-readable schema path used in assertion failure messages. + + Returns + ------- + list[Any] + Nested sequence value. + + Raises + ------ + pytest.fail.Exception + Raised when the key is missing or the value is not a sequence. + + Examples + -------- + Extract workflow steps from a parsed job:: + + steps = require_sequence(job, "steps", "CI lint-test job") + """ + value = mapping.get(key) + if not isinstance(value, list): + pytest.fail(f"expected {label} to include sequence key {key!r}") + return value diff --git a/tests/helpers/makefile_contracts.py b/tests/helpers/makefile_contracts.py new file mode 100644 index 0000000..9e579c9 --- /dev/null +++ b/tests/helpers/makefile_contracts.py @@ -0,0 +1,142 @@ +"""Validate rendered Makefile contracts and parsed command recipes. + +This module owns Makefile-specific assertions for generated projects. The +AGENTS contract helper imports ``_parse_makefile_rules`` from here so +documented commands can be compared against the parsed recipes without keeping +all contract domains in one large module. +""" + +from __future__ import annotations + +import re +import tempfile +from pathlib import Path + +import make_parser + + +def assert_common_make_targets(makefile: str) -> None: + """Assert Makefile targets shared by all generated variants. + + Parameters + ---------- + makefile : str + UTF-8 text of a generated Makefile. + + Returns + ------- + None + The helper returns after all shared Makefile assertions pass. + + Raises + ------ + AssertionError + Raised when a shared generated Makefile target or cleanup path is + missing. + + Examples + -------- + Validate shared targets after reading a generated Makefile:: + + assert_common_make_targets(makefile) + """ + assert "lint-python: build" in makefile, "Makefile should expose lint-python" + assert "lint: lint-python" in makefile, "lint should delegate to lint-python" + assert "audit: build" in makefile, "Makefile should expose audit" + assert ".uv-cache .uv-tools" in makefile, "clean should remove uv state dirs" + + +def _parse_makefile_rules(makefile: str) -> dict[str, list[str]]: + """Return generated Makefile rules parsed through make-parser.""" + target_names = set( + re.findall(r"^([a-zA-Z][a-zA-Z_-]*):", makefile, flags=re.MULTILINE) + ) + normalised_targets = { + target: target.replace("-", "_") for target in target_names if "-" in target + } + + def normalise_target(match: re.Match[str]) -> str: + target = match.group(1) + return normalised_targets.get(target, target) + ":" + + normalised_makefile = re.sub( + r"^([a-zA-Z][a-zA-Z_-]*):", + normalise_target, + makefile.replace("?=", "="), + flags=re.MULTILINE, + ) + with tempfile.TemporaryDirectory() as tmp_dir: + makefile_path = Path(tmp_dir) / "Makefile" + makefile_path.write_text(normalised_makefile, encoding="utf-8") + parsed = make_parser.make_load(makefile_path) + normalised_rules = parsed["rules"] + return { + target: normalised_rules[normalised_targets.get(target, target)]["commands"] + for target in target_names + if normalised_targets.get(target, target) in normalised_rules + } + + +def _assert_makefile_contracts(*, makefile: str, use_rust: bool) -> None: + """Assert generated Makefile contracts for both template variants.""" + assert_common_make_targets(makefile) + assert "WITH_ACT ?= 0" in makefile, ( + "expected generated Makefile to default act validation off" + ) + assert "ACT_TEST_ENV =" in makefile, ( + "expected generated Makefile to map WITH_ACT to pytest environment" + ) + assert "RUN_ACT_VALIDATION=1" in makefile, ( + "expected generated Makefile to enable act validation for pytest" + ) + assert "$(UV_ENV) $(ACT_TEST_ENV) $(UV) run pytest" in makefile, ( + "expected generated test target to include the act test environment" + ) + assert "PYTHON_TARGETS ?=" in makefile, ( + "expected generated Makefile to define Python target selection" + ) + assert "PYLINT_PYPY_SHIM_REF ?=" in makefile, ( + "expected generated Makefile to expose the Pylint shim revision" + ) + assert "test: build $(VENV_TOOLS)" in makefile, ( + "expected generated Makefile test target to depend on the project env" + ) + assert "$(UV_ENV) $(UV) run pip-audit" in makefile, ( + "expected generated audit target to run pip-audit" + ) + if use_rust: + assert "TEST_CMD :=" in makefile, ( + "expected Rust variant to select nextest or cargo test" + ) + assert "lint-rust: build whitaker" in makefile, ( + "expected Rust variant to expose the Rust lint target" + ) + assert "cargo is required for Rust tests" in makefile, ( + "expected Rust variant to fail clearly without cargo" + ) + assert "$(CARGO) $(TEST_CMD) $(TEST_FLAGS)" in makefile, ( + "expected Rust variant tests to use the selected cargo test command" + ) + assert "ifneq ($(TEST_CMD),test)" not in makefile, ( + "expected Rust variant tests to run doctests even when nextest is absent" + ) + assert ( + 'RUSTFLAGS="$(RUST_FLAGS)" $(CARGO) test --doc ' + "--manifest-path $(RUST_CRATE_DIR)/Cargo.toml --all-features" + ) in makefile, "expected Rust variant tests to run Rust doctests" + assert "rust-audit:" in makefile, ( + "expected Rust variant to expose the rust-audit target" + ) + assert "cd $(RUST_CRATE_DIR) && $(CARGO) audit" in makefile, ( + "expected Rust variant audit target to run cargo audit" + ) + else: + assert "lint-rust" not in makefile, ( + "expected pure-Python variant to omit Rust lint targets" + ) + assert "TEST_CMD :=" not in makefile, ( + "expected pure-Python variant to omit Rust test command selection" + ) + assert "rust-audit" not in makefile, ( + "expected pure-Python variant to omit Rust audit targets" + ) diff --git a/tests/helpers/pyproject_contracts.py b/tests/helpers/pyproject_contracts.py new file mode 100644 index 0000000..7e32024 --- /dev/null +++ b/tests/helpers/pyproject_contracts.py @@ -0,0 +1,74 @@ +"""Validate rendered pyproject packaging contracts. + +This module contains pyproject-specific assertions extracted from +``tests.helpers.tooling_contracts``. The top-level tooling-contract +orchestrator imports this private helper so packaging checks remain isolated +from Makefile and workflow assertions while sharing the generated-file parsing +helpers. +""" + +from __future__ import annotations + +from typing import Any + +from tests.helpers.generated_files import require_mapping + + +def _assert_pyproject_contracts( + *, package_name: str, pyproject: dict[str, Any], use_rust: bool +) -> None: + """Assert generated Python packaging contracts.""" + project = require_mapping(pyproject, "project", "pyproject.toml") + assert project.get("name"), "expected generated project metadata to include a name" + assert project.get("requires-python") == ">=3.10", ( + "expected generated pyproject.toml to use the requested Python version" + ) + dependency_groups = require_mapping( + pyproject, + "dependency-groups", + "pyproject.toml", + ) + dev_dependencies = dependency_groups.get("dev") + assert isinstance(dev_dependencies, list), ( + "expected generated pyproject.toml to include a dev dependency group" + ) + for dependency in ["pytest", "pip-audit", "ruff", "pyright", "ty", "pytest-xdist"]: + assert dependency in dev_dependencies, ( + f"expected generated dev dependencies to include {dependency}" + ) + pytest_options = require_mapping( + require_mapping(pyproject, "tool", "pyproject.toml"), + "pytest", + "pyproject.toml tool", + ) + pytest_ini_options = require_mapping( + pytest_options, + "ini_options", + "pyproject.toml tool.pytest", + ) + assert pytest_ini_options.get("testpaths") == ["tests"], ( + "expected generated pytest discovery to be limited to the tests tree" + ) + + build_system = require_mapping(pyproject, "build-system", "pyproject.toml") + if use_rust: + assert build_system.get("build-backend") == "maturin", ( + "expected Rust variant to use maturin as the build backend" + ) + maturin = require_mapping(pyproject, "tool", "pyproject.toml").get("maturin") + assert isinstance(maturin, dict), ( + "expected Rust variant to include tool.maturin configuration" + ) + assert maturin.get("manifest-path") == "rust_extension/Cargo.toml", ( + "expected Rust variant to point maturin at the extension manifest" + ) + assert maturin.get("module-name") == f"{package_name}._{package_name}_rs", ( + "expected Rust variant to render the package-specific module name" + ) + else: + assert build_system.get("build-backend") == "hatchling.build", ( + "expected pure-Python variant to use hatchling as the build backend" + ) + assert "maturin" not in pyproject.get("tool", {}), ( + "expected pure-Python variant to omit maturin configuration" + ) diff --git a/tests/helpers/release_contracts.py b/tests/helpers/release_contracts.py new file mode 100644 index 0000000..9bb0842 --- /dev/null +++ b/tests/helpers/release_contracts.py @@ -0,0 +1,62 @@ +"""Validate rendered release workflow contracts. + +Release workflow assertions live here to keep release-specific job wiring and +asset-upload checks separate from CI, Makefile, and wheel-action contracts. +The helper imports the shared checkout credential assertion from +``tests.helpers.ci_contracts``. +""" + +from __future__ import annotations + +from tests.helpers.ci_contracts import _checkout_steps_disable_credentials +from tests.helpers.generated_files import ( + parse_yaml_mapping, + require_mapping, + require_sequence, +) + + +def _assert_release_workflow_contracts( + *, release_workflow: str, use_rust: bool +) -> None: + """Assert generated release workflow contracts.""" + parsed_release_workflow = parse_yaml_mapping(release_workflow, "release workflow") + jobs = require_mapping(parsed_release_workflow, "jobs", "release workflow") + release = require_mapping(jobs, "release", "release workflow jobs") + release_steps = require_sequence(release, "steps", "release job") + assert _checkout_steps_disable_credentials(release_steps), ( + "expected generated release workflow to disable checkout credentials" + ) + assert "softprops/action-gh-release@v2" in release_workflow, ( + "expected release workflow to create a GitHub release" + ) + assert "actions/download-artifact@v4" in release_workflow, ( + "expected release workflow to download wheel artefacts" + ) + assert "--clobber" in release_workflow, ( + "expected release workflow to overwrite existing wheel assets on rerun" + ) + if use_rust: + assert "build-wheels" in jobs, ( + "expected Rust variant release workflow to build platform wheels" + ) + build_wheels = require_mapping(jobs, "build-wheels", "release workflow jobs") + assert build_wheels.get("uses") == "./.github/workflows/build-wheels.yml", ( + "expected Rust variant release workflow to call build-wheels.yml" + ) + assert release.get("needs") == ["build-wheels"], ( + "expected Rust variant release job to wait for platform wheels" + ) + assert "pure-wheel" not in jobs, ( + "expected Rust variant release workflow to skip pure wheel job" + ) + else: + assert "pure-wheel" in jobs, ( + "expected pure-Python release workflow to build one pure wheel" + ) + assert release.get("needs") == ["pure-wheel"], ( + "expected pure-Python release job to wait for the pure wheel" + ) + assert "build-wheels" not in jobs, ( + "expected pure-Python release workflow to skip platform wheel matrix" + ) diff --git a/tests/helpers/rendering.py b/tests/helpers/rendering.py new file mode 100644 index 0000000..2409931 --- /dev/null +++ b/tests/helpers/rendering.py @@ -0,0 +1,141 @@ +"""Render Copier projects and bridge generated-file helper APIs. + +This module wraps ``pytest-copier`` interactions used by template tests: +rendering projects, running generated quality gates, importing generated +packages, and reading rendered files relative to a project root. File reads +delegate to :mod:`tests.helpers.generated_files` so rendering-oriented tests +share the same pytest failure semantics as lower-level generated-file parsers. +Tooling contract tests call these helpers before passing rendered text into +:mod:`tests.helpers.tooling_contracts`. +""" + +from __future__ import annotations + +import shlex +from pathlib import Path + +from pytest_copier.plugin import CopierFixture, CopierProject + +from tests.helpers.generated_files import read_generated_text + + +def run_quality_gates(project: CopierProject) -> None: + """Run the rendered project's public quality gate. + + Parameters + ---------- + project : CopierProject + Rendered ``pytest-copier`` project whose root contains the generated + Makefile. + + Returns + ------- + None + The helper returns after the generated ``make all`` target succeeds. + + Raises + ------ + AssertionError + Raised by ``pytest-copier`` when the generated command exits + unsuccessfully. + """ + project.run("make all") + + +def render_project( + tmp_path: Path, + copier: CopierFixture, + *, + project_name: str, + package_name: str, + use_rust: bool = False, + python_version: str = "3.10", +) -> CopierProject: + """Render a generated Python project with explicit template answers. + + Parameters + ---------- + tmp_path : pathlib.Path + Temporary directory used as the generated project destination. + copier : CopierFixture + ``pytest-copier`` fixture bound to this template repository. + project_name : str + Project name answer passed to Copier. + package_name : str + Python import package name answer passed to Copier. + use_rust : bool, default=False + Whether to include the optional PyO3 extension. + python_version : str, default="3.10" + Minimum supported Python version answer passed to Copier. + + Returns + ------- + CopierProject + Rendered project wrapper for file assertions and command execution. + + Raises + ------ + Exception + Propagates rendering failures raised by Copier or ``pytest-copier``. + """ + return copier.copy( + tmp_path, + project_name=project_name, + package_name=package_name, + use_rust=use_rust, + python_version=python_version, + ) + + +def check_generated_import(project: CopierProject, package: str, greeting: str) -> None: + """Import a generated package and assert its greeting. + + Parameters + ---------- + project : CopierProject + Rendered project whose managed environment should contain the package. + package : str + Import name to load through ``importlib.import_module``. + greeting : str + Expected return value from the generated package's ``hello`` function. + + Returns + ------- + None + The helper returns when the generated import and assertion succeed. + + Raises + ------ + AssertionError + Raised by ``pytest-copier`` when the generated command exits + unsuccessfully. + """ + script = ( + "import importlib; " + f"module = importlib.import_module({package!r}); " + f"assert module.hello() == {greeting!r}" + ) + project.run(f"uv run python -c {shlex.quote(script)}") + + +def read_generated_file(project: CopierProject, relative_path: str) -> str: + """Read a rendered project file as UTF-8 text. + + Parameters + ---------- + project : CopierProject + Rendered ``pytest-copier`` project to read from. + relative_path : str + Path to the generated file, relative to the rendered project root. + + Returns + ------- + str + File contents decoded as UTF-8 text. + + Raises + ------ + pytest.fail.Exception + Raised when the requested generated file cannot be read. + """ + return read_generated_text(project / relative_path) diff --git a/tests/helpers/tooling_contracts.py b/tests/helpers/tooling_contracts.py new file mode 100644 index 0000000..19569a9 --- /dev/null +++ b/tests/helpers/tooling_contracts.py @@ -0,0 +1,141 @@ +"""Orchestrate rendered tooling contracts for generated project variants. + +This module keeps the public test-helper API stable while delegating +domain-specific assertions to focused helper modules: +``pyproject_contracts``, ``agents_contracts``, ``makefile_contracts``, +``ci_contracts``, ``release_contracts``, and ``wheel_contracts``. Top-level +template tests import from here, while each delegated module stays small enough +to own one contract domain. +""" + +from __future__ import annotations + +from typing import Any + +from tests.helpers.agents_contracts import ( + _assert_agents_contracts, + _assert_agents_make_targets_mirror_makefile, +) +from tests.helpers.ci_contracts import ( + _assert_act_validation_workflow_contracts, + _assert_ci_workflow_contracts, + _parse_ci_workflow, + assert_ci_coverage_action_contract, +) +from tests.helpers.makefile_contracts import ( + _assert_makefile_contracts, + assert_common_make_targets, +) +from tests.helpers.pyproject_contracts import _assert_pyproject_contracts +from tests.helpers.release_contracts import _assert_release_workflow_contracts +from tests.helpers.wheel_contracts import _assert_wheel_workflow_contracts + +__all__ = [ + "assert_ci_coverage_action_contract", + "assert_common_make_targets", + "assert_generated_tooling_contracts", +] + + +def assert_generated_tooling_contracts( + *, + package_name: str, + agents: str, + pyproject: dict[str, Any], + makefile: str, + ci_workflow: str, + act_validation_workflow: str, + release_workflow: str, + build_wheels_workflow: str, + build_wheels_action: str, + pure_wheel_action: str, + use_rust: bool, +) -> None: + """Assert generated Python/Rust tooling contracts from one validator. + + Parameters + ---------- + package_name : str + Generated Python import package name. + agents : str + UTF-8 text of the generated ``AGENTS.md`` file. + pyproject : dict[str, Any] + Parsed generated ``pyproject.toml`` mapping. + makefile : str + UTF-8 text of the generated Makefile. + ci_workflow : str + UTF-8 text of the generated CI workflow. + act_validation_workflow : str + UTF-8 text of the generated act-validation workflow. + release_workflow : str + UTF-8 text of the generated release workflow. + build_wheels_workflow : str + UTF-8 text of the generated build-wheels workflow. + build_wheels_action : str + UTF-8 text of the generated build-wheels composite action. + pure_wheel_action : str + UTF-8 text of the generated pure-wheel composite action. + use_rust : bool + Whether the rendered variant includes the optional Rust extension. + + Returns + ------- + None + The helper returns after all generated tooling contracts pass. + + Raises + ------ + AssertionError + Raised when any generated tooling, workflow, or packaging contract is + missing or variant-inconsistent. + + Examples + -------- + Validate generated contracts after rendering a project:: + + assert_generated_tooling_contracts( + package_name="example_pkg", + agents=agents, + pyproject=pyproject, + makefile=makefile, + ci_workflow=ci_workflow, + act_validation_workflow=act_validation_workflow, + release_workflow=release_workflow, + build_wheels_workflow=build_wheels_workflow, + build_wheels_action=build_wheels_action, + pure_wheel_action=pure_wheel_action, + use_rust=False, + ) + """ + parsed_ci_workflow = _parse_ci_workflow(ci_workflow) + _assert_pyproject_contracts( + package_name=package_name, + pyproject=pyproject, + use_rust=use_rust, + ) + _assert_agents_contracts(agents) + _assert_agents_make_targets_mirror_makefile( + agents=agents, + makefile=makefile, + package_name=package_name, + use_rust=use_rust, + ) + _assert_makefile_contracts(makefile=makefile, use_rust=use_rust) + _assert_ci_workflow_contracts( + parsed_ci_workflow=parsed_ci_workflow, + ci_workflow=ci_workflow, + use_rust=use_rust, + ) + _assert_act_validation_workflow_contracts( + act_validation_workflow=act_validation_workflow, + use_rust=use_rust, + ) + _assert_release_workflow_contracts( + release_workflow=release_workflow, + use_rust=use_rust, + ) + _assert_wheel_workflow_contracts( + build_wheels_workflow=build_wheels_workflow, + build_wheels_action=build_wheels_action, + pure_wheel_action=pure_wheel_action, + ) diff --git a/tests/helpers/wheel_contracts.py b/tests/helpers/wheel_contracts.py new file mode 100644 index 0000000..1fbe895 --- /dev/null +++ b/tests/helpers/wheel_contracts.py @@ -0,0 +1,50 @@ +"""Validate rendered wheel workflow and composite action contracts. + +Wheel-building assertions are separated from release workflow checks so matrix +coverage, checkout hardening, and build-command contracts remain focused on the +workflow/action files that produce wheel artefacts. +""" + +from __future__ import annotations + +from tests.helpers.generated_files import ( + parse_yaml_mapping, + require_mapping, + require_sequence, +) + + +def _assert_wheel_workflow_contracts( + *, + build_wheels_workflow: str, + build_wheels_action: str, + pure_wheel_action: str, +) -> None: + """Assert generated wheel workflow and action contracts.""" + parsed_build_wheels = parse_yaml_mapping( + build_wheels_workflow, + "build-wheels workflow", + ) + build_jobs = require_mapping(parsed_build_wheels, "jobs", "build-wheels workflow") + build_job = require_mapping(build_jobs, "build", "build-wheels workflow jobs") + strategy = require_mapping(build_job, "strategy", "build-wheels build job") + matrix = require_mapping(strategy, "matrix", "build-wheels strategy") + includes = require_sequence(matrix, "include", "build-wheels matrix") + assert len(includes) == 6, ( + "expected build-wheels workflow to cover Linux, Windows, and macOS" + ) + assert "persist-credentials: false" in build_wheels_workflow, ( + "expected build-wheels workflow checkout to disable credentials" + ) + assert "uvx --with 'cibuildwheel>=2.16.0,<4.0.0' cibuildwheel" in ( + build_wheels_action + ), "expected Rust wheel action to build through cibuildwheel" + assert "persist-credentials: false" in build_wheels_action, ( + "expected build-wheels action checkout to disable credentials" + ) + assert "uv build --wheel" in pure_wheel_action, ( + "expected pure-wheel action to build through uv" + ) + assert "persist-credentials: false" in pure_wheel_action, ( + "expected pure-wheel action checkout to disable credentials" + ) diff --git a/tests/test_audit.py b/tests/test_audit.py new file mode 100644 index 0000000..c229ea7 --- /dev/null +++ b/tests/test_audit.py @@ -0,0 +1,135 @@ +"""Validate rendered dependency audit Makefile targets. + +This module exercises the generated ``make audit`` target for pure-Python and +Python/Rust renders without contacting external vulnerability databases. The +tests use fake ``uv`` and ``cargo`` executables to record the audit commands +that the rendered Makefile dispatches. +""" + +from __future__ import annotations + +import shutil +import subprocess +from pathlib import Path + +import pytest +from pytest_copier.plugin import CopierFixture + +from tests.helpers.rendering import render_project + + +@pytest.mark.parametrize( + ("target_dir", "project_name", "package_name", "use_rust"), + [ + ("audit-pure", "AuditPure", "audit_pure", False), + ("audit-rust", "AuditRust", "audit_rust", True), + ], +) +def test_generated_audit_target_runs_expected_tools( + copier: CopierFixture, + tmp_path: Path, + target_dir: str, + project_name: str, + package_name: str, + use_rust: bool, +) -> None: + """Validate generated audit target command dispatch. + + Parameters + ---------- + copier : CopierFixture + Fixture used to render the template into a temporary project. + tmp_path : Path + Temporary directory used for the rendered project and fake tools. + target_dir : str + Temporary project directory name for the rendered variant. + project_name : str + Project name answer passed to Copier. + package_name : str + Package name answer passed to Copier. + use_rust : bool + Whether the rendered variant includes the optional Rust extension. + + Returns + ------- + None + The test passes when ``make audit`` runs ``pip-audit`` for every + variant and runs ``cargo audit`` only for Rust-enabled projects. + """ + project = render_project( + tmp_path / target_dir, + copier, + project_name=project_name, + package_name=package_name, + use_rust=use_rust, + ) + command_log = tmp_path / f"{target_dir}-audit.log" + fake_uv = _write_fake_uv(tmp_path / f"{target_dir}-bin", command_log) + fake_cargo = _write_fake_cargo(tmp_path / f"{target_dir}-cargo", command_log) + make = shutil.which("make") + assert make is not None, "expected make to be available for generated tests" + + result = subprocess.run( + [make, "audit", f"UV={fake_uv}", f"CARGO={fake_cargo}"], + cwd=project.path, + check=False, + capture_output=True, + text=True, + ) + + output = f"{result.stdout}\n{result.stderr}" + assert result.returncode == 0, output + log_lines = command_log.read_text(encoding="utf-8").splitlines() + assert "uv|pip-audit" in log_lines, ( + "expected generated audit target to run pip-audit through uv" + ) + if use_rust: + assert f"cargo|{project.path / 'rust_extension'}|audit" in log_lines, ( + "expected Rust-enabled audit target to run cargo audit in rust_extension" + ) + else: + assert all(not line.startswith("cargo|") for line in log_lines), ( + "expected pure-Python audit target to omit cargo audit" + ) + + +def _write_fake_uv(bin_dir: Path, command_log: Path) -> Path: + """Write a fake uv executable that records audit invocations.""" + bin_dir.mkdir(parents=True, exist_ok=True) + uv_path = bin_dir / "uv" + uv_path.write_text( + "#!/usr/bin/env sh\n" + 'if [ "$1" = venv ]; then\n' + " mkdir -p .venv/bin\n" + " exit 0\n" + "fi\n" + 'if [ "$1" = sync ]; then\n' + " exit 0\n" + "fi\n" + 'if [ "$1" = run ]; then\n' + " shift\n" + f" printf 'uv|%s\\n' \"$*\" >> '{command_log}'\n" + " exit 0\n" + "fi\n" + "exit 0\n", + encoding="utf-8", + ) + uv_path.chmod(0o755) + return uv_path + + +def _write_fake_cargo(bin_dir: Path, command_log: Path) -> Path: + """Write a fake cargo executable that records audit invocations.""" + bin_dir.mkdir(parents=True, exist_ok=True) + cargo_path = bin_dir / "cargo" + cargo_path.write_text( + "#!/usr/bin/env sh\n" + 'if [ "$1" = audit ]; then\n' + f" printf 'cargo|%s|%s\\n' \"$PWD\" \"$*\" >> '{command_log}'\n" + " exit 0\n" + "fi\n" + "exit 0\n", + encoding="utf-8", + ) + cargo_path.chmod(0o755) + return cargo_path diff --git a/tests/test_github_actions_integration.py b/tests/test_github_actions_integration.py index e68509d..0a840ad 100644 --- a/tests/test_github_actions_integration.py +++ b/tests/test_github_actions_integration.py @@ -26,6 +26,8 @@ import json import subprocess from pathlib import Path +from types import SimpleNamespace +from typing import Any, cast import pytest from pytest_copier.plugin import CopierFixture, CopierProject @@ -36,6 +38,7 @@ ACT_IMAGE = "ubuntu-latest=catthehacker/ubuntu:act-latest" GENERATE_COVERAGE_STEP = "Test and Measure Coverage" + def prepare_git_repository(project: CopierProject) -> None: """Initialise a rendered project as a Git repository for act. @@ -118,6 +121,9 @@ def run_act(project: CopierProject, *, artifact_dir: Path) -> tuple[int, str]: docker_host = container_daemon_socket(env) if docker_host is not None: command.extend(["--container-daemon-socket", docker_host]) + act_github_token = env.get("ACT_GITHUB_TOKEN") + if act_github_token: + command.extend(["-s", f"GITHUB_TOKEN={act_github_token}"]) completed = subprocess.run( command, cwd=project.path, @@ -130,6 +136,76 @@ def run_act(project: CopierProject, *, artifact_dir: Path) -> tuple[int, str]: return completed.returncode, f"{completed.stdout}\n{completed.stderr}" +@pytest.mark.parametrize( + ("env", "expected_secret"), + [ + ({}, None), + ({"ACT_GITHUB_TOKEN": "nested-token"}, "GITHUB_TOKEN=nested-token"), + ], +) +def test_run_act_forwards_only_explicit_act_github_token( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + env: dict[str, str], + expected_secret: str | None, +) -> None: + """Forward only explicit nested-act GitHub tokens. + + Parameters + ---------- + monkeypatch + Pytest fixture used to replace subprocess and environment helpers. + tmp_path + Temporary directory used as the fake rendered project and artifact + location. + env + Sanitized act subprocess environment returned by ``docker_environment``. + expected_secret + Expected ``GITHUB_TOKEN`` secret argument, or ``None`` when no secret + should be passed to act. + + Returns + ------- + None + The test passes when ``run_act`` forwards ``ACT_GITHUB_TOKEN`` as an act + secret and does not synthesize a secret when it is absent. + """ + captured_command: list[str] = [] + + def fake_run(command: list[str], **_: Any) -> subprocess.CompletedProcess[str]: + captured_command.extend(command) + return subprocess.CompletedProcess(command, 0, "stdout", "stderr") + + monkeypatch.setattr( + "tests.test_github_actions_integration.docker_environment", + lambda: env, + ) + monkeypatch.setattr( + "tests.test_github_actions_integration.container_daemon_socket", + lambda _: None, + ) + monkeypatch.setattr(subprocess, "run", fake_run) + project = cast("CopierProject", SimpleNamespace(path=tmp_path)) + + run_act(project, artifact_dir=tmp_path / "artifacts") + + if expected_secret is None: + assert "-s" not in captured_command, ( + "expected run_act not to pass act secrets without ACT_GITHUB_TOKEN" + ) + assert not any( + argument.startswith("GITHUB_TOKEN=") for argument in captured_command + ), "expected run_act not to synthesize a GITHUB_TOKEN secret" + else: + assert "-s" in captured_command, ( + "expected run_act to pass an act secret when ACT_GITHUB_TOKEN is set" + ) + secret_index = captured_command.index("-s") + 1 + assert captured_command[secret_index] == expected_secret, ( + "expected run_act to forward ACT_GITHUB_TOKEN as GITHUB_TOKEN secret" + ) + + def iter_json_log_events(logs: str) -> list[dict[str, object]]: """Return JSON event objects from an act log stream. @@ -234,12 +310,8 @@ def assert_ci_exercised_expected_steps(logs: str, *, use_rust: bool) -> None: saw_python = False saw_rust = not use_rust for event in iter_json_log_events(logs): - output = str( - event_text(event, "Output", "output", "message", "msg") - ) - step = str( - event_text(event, "name", "step_name", "Step", "step") - ) + output = str(event_text(event, "Output", "output", "message", "msg")) + step = str(event_text(event, "name", "step_name", "Step", "step")) in_coverage_step = GENERATE_COVERAGE_STEP in step saw_coverage = saw_coverage or ( in_coverage_step @@ -296,12 +368,7 @@ def assert_act_result( assert_act_result(project, code, logs, use_rust=False) """ - assert ( - project / "coverage.xml" - ).exists(), "act workflow should write coverage.xml in the generated project" assert_ci_exercised_expected_steps(logs, use_rust=use_rust) - if code == 0: - return if ( "Parameter INPUT_ARTEFACT_NAME_SUFFIX specified multiple times" in logs and "Provided artifact name input during validation is empty" in logs @@ -311,6 +378,9 @@ def assert_act_result( "action output/archive phase after tests and coverage succeed" ) assert code == 0, logs + assert (project / "coverage.xml").exists(), ( + "act workflow should write coverage.xml in the generated project" + ) @pytest.mark.parametrize( @@ -380,11 +450,11 @@ def test_generated_workflow_runs_with_shared_coverage_action( workflow = (project / ".github" / "workflows" / "ci.yml").read_text( encoding="utf-8" ) - assert ( - "leynos/shared-actions/.github/actions/generate-coverage" in workflow - ), "Generated workflow should use the shared generate-coverage action" + assert "leynos/shared-actions/.github/actions/generate-coverage" in workflow, ( + "Generated workflow should use the shared generate-coverage action" + ) if use_rust: - assert ( - "cargo-manifest: rust_extension/Cargo.toml" in workflow - ), "Rust workflow should pass the Rust extension manifest to coverage" + assert "cargo-manifest: rust_extension/Cargo.toml" in workflow, ( + "Rust workflow should pass the Rust extension manifest to coverage" + ) assert_act_result(project, code, logs, use_rust=use_rust) diff --git a/tests/test_helpers.py b/tests/test_helpers.py new file mode 100644 index 0000000..976a7fe --- /dev/null +++ b/tests/test_helpers.py @@ -0,0 +1,506 @@ +"""Validate direct helper-module error handling and edge cases. + +The tests in this module exercise support helpers without rendering a full +Copier project. They keep helper fallibility contracts explicit by checking +``pytest.fail`` conversion paths, generated-file schema helpers, and tooling +contract assertions directly. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path +from typing import TYPE_CHECKING, Any, cast + +import pytest + +from tests.conftest import MINIMUM_ACT_VERSION, _parse_act_version +from tests.helpers.generated_files import ( + parse_toml_file, + parse_yaml_mapping, + read_generated_text, + require_mapping, + require_sequence, +) +from tests.helpers.rendering import read_generated_file +from tests.helpers.tooling_contracts import ( + assert_ci_coverage_action_contract, + assert_common_make_targets, +) +from tests.utilities import docker_environment + +if TYPE_CHECKING: + from pytest_copier.plugin import CopierProject + + +REPO_ROOT = Path(__file__).resolve().parent.parent + + +@pytest.mark.parametrize( + ("output", "expected"), + [ + ("act version 0.2.84", (0, 2, 84)), + ("act version v0.2.84", (0, 2, 84)), + ], +) +def test_parse_act_version_accepts_supported_formats( + output: str, + expected: tuple[int, int, int], +) -> None: + """Parse act version output used by the local preflight. + + Parameters + ---------- + output + Text emitted by ``act --version``. + expected + Semantic version tuple expected from the parser. + + Returns + ------- + None + The test passes when supported act version output formats parse to the + expected tuple. + """ + assert _parse_act_version(output) == expected, ( + f"expected act version parser to parse {output!r}" + ) + + +def test_parse_act_version_rejects_unexpected_output() -> None: + """Reject act version output that does not contain a semantic version. + + Parameters + ---------- + None + This test does not use pytest fixtures. + + Returns + ------- + None + The test passes when unexpected output returns ``None`` so the preflight + can skip optional act-backed tests with a clear reason. + """ + assert _parse_act_version("unexpected act output") is None, ( + "expected act version parser to reject output without a semantic version" + ) + + +def test_old_act_version_is_below_minimum() -> None: + """Compare stale act versions against the Node24-capable minimum. + + Parameters + ---------- + None + This test does not use pytest fixtures. + + Returns + ------- + None + The test passes when act ``0.2.80`` is detected as older than the + minimum version required for Node24 action runtime support. + """ + parsed_version = _parse_act_version("act version 0.2.80") + + assert parsed_version is not None, "expected parser to read stale act version" + assert parsed_version < MINIMUM_ACT_VERSION, ( + "expected act 0.2.80 to be below the Node24-capable minimum" + ) + + +def test_docker_environment_removes_github_auth_tokens( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Remove host GitHub credentials from act subprocess environments. + + Parameters + ---------- + monkeypatch + Pytest fixture used to install representative host GitHub token + variables. + + Returns + ------- + None + The test passes when ``docker_environment`` removes GitHub auth tokens + so stale host credentials cannot break public action clones in ``act``. + """ + monkeypatch.setenv("GITHUB_TOKEN", "stale-token") + monkeypatch.setenv("GH_TOKEN", "stale-token") + + env = docker_environment() + + assert "GITHUB_TOKEN" not in env, ( + "expected docker_environment to remove host GITHUB_TOKEN" + ) + assert "GH_TOKEN" not in env, "expected docker_environment to remove host GH_TOKEN" + + +def test_read_generated_text_converts_os_errors(tmp_path: Path) -> None: + """Convert generated-file read errors into pytest failures. + + Parameters + ---------- + tmp_path + Temporary directory used to build a missing file path for + ``read_generated_text``. + + Returns + ------- + None + The test passes when ``read_generated_text`` raises + ``pytest.fail.Exception`` with path context instead of propagating the + raw ``FileNotFoundError`` from ``Path.read_text``. + """ + missing_path = tmp_path / "nonexistent_generated.txt" + + with pytest.raises(pytest.fail.Exception, match="could not read generated file"): + read_generated_text(missing_path) + + +def test_parse_toml_file_reports_decode_errors(tmp_path: Path) -> None: + """Convert generated TOML decode errors into pytest failures. + + Parameters + ---------- + tmp_path + Temporary directory used to write invalid TOML content. + + Returns + ------- + None + The test passes when invalid TOML raises ``pytest.fail.Exception`` with + generated-file context. + """ + pyproject = tmp_path / "pyproject.toml" + pyproject.write_text("[project\nname = 'broken'\n", encoding="utf-8") + + with pytest.raises(pytest.fail.Exception, match="could not parse generated TOML"): + parse_toml_file(pyproject) + + +def test_parse_yaml_mapping_reports_invalid_yaml() -> None: + """Convert generated YAML parser errors into pytest failures. + + Parameters + ---------- + None + This test does not use pytest fixtures. + + Returns + ------- + None + The test passes when invalid YAML raises ``pytest.fail.Exception`` with + the supplied label. + """ + with pytest.raises(pytest.fail.Exception, match="could not parse generated CI"): + parse_yaml_mapping("jobs: [unterminated", "CI") + + +def test_parse_yaml_mapping_requires_mapping_root() -> None: + """Reject generated YAML documents that do not parse to mappings. + + Parameters + ---------- + None + This test does not use pytest fixtures. + + Returns + ------- + None + The test passes when a sequence root raises ``pytest.fail.Exception``. + """ + with pytest.raises( + pytest.fail.Exception, + match="expected generated CI workflow to parse as a mapping", + ): + parse_yaml_mapping("- lint\n- test\n", "CI workflow") + + +def test_generated_file_schema_helpers_require_expected_shapes() -> None: + """Fail with schema-path context for wrong nested value shapes. + + Parameters + ---------- + None + This test does not use pytest fixtures. + + Returns + ------- + None + The test passes when mapping and sequence helpers accept valid values + and fail on missing or incorrectly typed values. + """ + mapping: dict[str, Any] = { + "jobs": {"lint-test": {}}, + "steps": [{"name": "Check"}], + } + + assert require_mapping(mapping, "jobs", "CI workflow") == {"lint-test": {}}, ( + "expected require_mapping(mapping, 'jobs', 'CI workflow') to return a " + "jobs mapping containing lint-test" + ) + assert require_sequence(mapping, "steps", "CI lint-test job") == [ + {"name": "Check"} + ], ( + "expected require_sequence(mapping, 'steps', 'CI lint-test job') to " + "return steps sequence [{'name': 'Check'}]" + ) + + with pytest.raises( + pytest.fail.Exception, + match="expected CI workflow to include mapping key 'jobs'", + ): + require_mapping({"jobs": []}, "jobs", "CI workflow") + + with pytest.raises( + pytest.fail.Exception, + match="expected CI lint-test job to include sequence key 'steps'", + ): + require_sequence({"steps": {}}, "steps", "CI lint-test job") + + +def test_read_generated_file_uses_shared_error_contract(tmp_path: Path) -> None: + """Read rendered files through the shared generated-file helper contract. + + Parameters + ---------- + tmp_path + Temporary rendered-project stand-in used as the project root. + + Returns + ------- + None + The test passes when existing files are read and missing files raise + ``pytest.fail.Exception`` instead of raw filesystem exceptions. + """ + generated = tmp_path / "docs" / "users-guide.md" + generated.parent.mkdir() + generated.write_text("generated docs\n", encoding="utf-8") + project = cast("CopierProject", tmp_path) + + assert read_generated_file(project, "docs/users-guide.md") == "generated docs\n", ( + "expected read_generated_file(project, 'docs/users-guide.md') to return " + "the generated docs text" + ) + with pytest.raises(pytest.fail.Exception, match="could not read generated file"): + read_generated_file(project, "missing.md") + + +def test_common_make_targets_reports_missing_contracts() -> None: + """Report missing shared Makefile targets through assertion messages. + + Parameters + ---------- + None + This test does not use pytest fixtures. + + Returns + ------- + None + The test passes when the shared target assertion accepts a complete + Makefile fragment and rejects a fragment missing required targets. + """ + assert_common_make_targets( + "lint-python: build\n" + "lint: lint-python\n" + "audit: build\n" + "clean:\n" + "\trm -rf .uv-cache .uv-tools\n" + ) + + with pytest.raises(AssertionError, match="Makefile should expose lint-python"): + assert_common_make_targets("lint: lint-python\n") + + +def test_ci_coverage_action_contract_validates_pure_python_edges() -> None: + """Validate pure-Python CI coverage action edge cases. + + Parameters + ---------- + None + This test does not use pytest fixtures. + + Returns + ------- + None + The test passes when a valid pure-Python workflow is accepted and a + workflow with persistent checkout credentials is rejected. + """ + assert_ci_coverage_action_contract( + ci_workflow=_ci_workflow( + persist_credentials="false", + coverage_inputs=" artefact-name-suffix: helper-pkg\n", + ), + package_name="helper_pkg", + use_rust=False, + ) + + with pytest.raises( + AssertionError, + match="expected CI checkout steps to disable credential persistence", + ): + assert_ci_coverage_action_contract( + ci_workflow=_ci_workflow( + persist_credentials="true", + coverage_inputs=" artefact-name-suffix: helper-pkg\n", + ), + package_name="helper_pkg", + use_rust=False, + ) + + +def test_ci_coverage_action_contract_validates_rust_manifest_edge() -> None: + """Validate Rust CI coverage action cargo-manifest requirements. + + Parameters + ---------- + None + This test does not use pytest fixtures. + + Returns + ------- + None + The test passes when the Rust workflow requires the extension + ``cargo-manifest`` input on the shared coverage action. + """ + with pytest.raises( + AssertionError, + match="expected Rust variant to pass the extension manifest to coverage", + ): + assert_ci_coverage_action_contract( + ci_workflow=_ci_workflow( + persist_credentials="false", + coverage_inputs=" artefact-name-suffix: helper-pkg\n", + ), + package_name="helper_pkg", + use_rust=True, + ) + + assert_ci_coverage_action_contract( + ci_workflow=_ci_workflow( + persist_credentials="false", + coverage_inputs=( + " artefact-name-suffix: helper-pkg\n" + " cargo-manifest: rust_extension/Cargo.toml\n" + ), + ), + package_name="helper_pkg", + use_rust=True, + ) + + +def test_parent_makefile_help_target_lists_available_targets() -> None: + """Validate the parent repository ``help`` target output. + + Parameters + ---------- + None + This test does not use pytest fixtures. + + Returns + ------- + None + The test passes when ``make help`` advertises the parent ``help`` and + quality-gate targets. + """ + result = subprocess.run( + ["make", "help"], + check=True, + capture_output=True, + cwd=REPO_ROOT, + encoding="utf-8", + ) + + assert "Available targets:" in result.stdout, ( + "expected parent Makefile help target to print an available-targets header" + ) + assert "help" in result.stdout, ( + "expected parent Makefile help target to list the help target" + ) + for target in ["check-fmt", "lint", "typecheck", "test"]: + assert target in result.stdout, ( + f"expected parent Makefile help target to list the {target} target" + ) + + +def test_parent_makefile_test_target_uses_requisite_pytest_command() -> None: + """Validate the parent repository ``test`` target command contract. + + Parameters + ---------- + None + This test does not use pytest fixtures. + + Returns + ------- + None + The test passes when the parent Makefile exposes ``test`` as phony, + checks for ``uvx``, and runs pytest through the resolved executable with + the required template-test packages. + """ + makefile = (REPO_ROOT / "Makefile").read_text(encoding="utf-8") + + assert ".PHONY: help check-fmt lint test typecheck" in makefile, ( + "expected parent Makefile to mark documented gate targets as phony" + ) + assert "check-fmt: ## Verify template test formatting" in makefile, ( + "expected parent Makefile to expose a documented check-fmt target" + ) + assert "$(UV) ruff format --check tests/" in makefile, ( + "expected parent Makefile check-fmt target to run Ruff formatting checks" + ) + assert "lint: ## Run template test lint checks" in makefile, ( + "expected parent Makefile to expose a documented lint target" + ) + assert "$(UV) ruff check tests/" in makefile, ( + "expected parent Makefile lint target to run Ruff checks" + ) + assert "typecheck: ## Run template test type checks" in makefile, ( + "expected parent Makefile to expose a documented typecheck target" + ) + assert ( + "$(UV) --with pytest --with pytest-copier --with pyyaml --with syrupy " + "--with make-parser ty check tests/" in makefile + ), "expected parent Makefile typecheck target to run ty with test dependencies" + assert "test: ## Run template tests" in makefile, ( + "expected parent Makefile to expose a documented test target" + ) + assert "UV := $(shell command -v uvx 2>/dev/null)" in makefile, ( + "expected parent Makefile to resolve uvx before running tests" + ) + assert "uvx is required to run template tests" in makefile, ( + "expected parent Makefile to fail early with a uvx installation message" + ) + assert "WITH_ACT ?= 0" in makefile, ( + "expected parent Makefile to default act validation off" + ) + assert "RUN_ACT_VALIDATION=1" in makefile, ( + "expected parent Makefile to map WITH_ACT to act validation" + ) + assert ( + "$(ACT_TEST_ENV) $(UV) --with pytest-copier --with pyyaml --with syrupy " + "--with make-parser pytest tests/" in makefile + ), ( + "expected parent Makefile test target to run pytest through $(UV) with " + "pytest-copier, pyyaml, syrupy, make-parser, and act environment wiring" + ) + + +def _ci_workflow(*, persist_credentials: str, coverage_inputs: str) -> str: + """Return a minimal generated CI workflow for coverage-contract tests.""" + return f"""\ +name: CI +jobs: + lint-test: + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: {persist_credentials} + - name: Test and Measure Coverage + uses: leynos/shared-actions/.github/actions/generate-coverage@455d9ed03477c0026da96c2541ca26569a74acac + with: + output-path: coverage.xml + format: cobertura +{coverage_inputs}\ +""" diff --git a/tests/test_parent_ci.py b/tests/test_parent_ci.py new file mode 100644 index 0000000..19aaf2c --- /dev/null +++ b/tests/test_parent_ci.py @@ -0,0 +1,93 @@ +"""Validate parent repository CI workflow contracts.""" + +from __future__ import annotations + +from pathlib import Path + +from tests.helpers.generated_files import read_generated_text + +REPO_ROOT = Path(__file__).resolve().parent.parent + + +def test_parent_ci_splits_application_and_act_validation_tests() -> None: + """Validate parent CI splits normal and act-enabled test gates. + + Parameters + ---------- + None + This test does not use pytest fixtures. + + Returns + ------- + None + The test passes when the parent CI workflow runs normal tests and the + separate act-validation workflow installs act prerequisites before + running ``make test WITH_ACT=1``. + """ + ci_workflow = read_generated_text(REPO_ROOT / ".github" / "workflows" / "ci.yml") + act_workflow = read_generated_text( + REPO_ROOT / ".github" / "workflows" / "act-validation.yml" + ) + + assert "permissions:\n contents: read" in ci_workflow, ( + "expected parent CI to restrict GITHUB_TOKEN to repository contents reads" + ) + assert "MARKDOWNLINT_CLI2_VERSION: 0.22.1" in ci_workflow, ( + "expected parent CI to pin markdownlint-cli2" + ) + assert "MBAKE_VERSION: 1.4.6" in ci_workflow, "expected parent CI to pin mbake" + assert 'npm install -g "markdownlint-cli2@${MARKDOWNLINT_CLI2_VERSION}"' in ( + ci_workflow + ), "expected parent CI to install pinned markdownlint-cli2" + assert 'uv tool install "mbake==${MBAKE_VERSION}"' in ci_workflow, ( + "expected parent CI to install pinned mbake" + ) + assert "make test\n" in ci_workflow, ( + "expected parent CI to run the normal parent test gate" + ) + assert "uv tool install mdformat-all" not in ci_workflow, ( + "expected parent CI not to install mdformat-all through uv" + ) + assert "mdtablefix" not in ci_workflow, ( + "expected parent CI not to install Markdown formatting tools" + ) + assert "make test WITH_ACT=1" not in ci_workflow, ( + "expected parent CI to leave act validation to a separate workflow" + ) + assert "permissions:\n contents: read" in act_workflow, ( + "expected parent act-validation workflow to restrict GITHUB_TOKEN" + ) + assert "ACT_VERSION:" in act_workflow, ( + "expected parent act-validation workflow to declare an act version" + ) + assert "act_Linux_x86_64.tar.gz" in act_workflow, ( + "expected parent act-validation workflow to include act_Linux_x86_64.tar.gz" + ) + assert "${ACT_VERSION}" in act_workflow, ( + "expected parent act-validation workflow to include ${ACT_VERSION}" + ) + assert "sha256sum -c -" in act_workflow, ( + "expected parent act-validation workflow to verify the act archive checksum" + ) + assert 'npm install -g "markdownlint-cli2@${MARKDOWNLINT_CLI2_VERSION}"' in ( + act_workflow + ), "expected parent act-validation workflow to install pinned markdownlint-cli2" + assert 'uv tool install "mbake==${MBAKE_VERSION}"' in act_workflow, ( + "expected parent act-validation workflow to install pinned mbake" + ) + assert "uv tool install mdformat-all" not in act_workflow, ( + "expected parent act-validation workflow not to install mdformat-all through uv" + ) + assert "mdtablefix" not in act_workflow, ( + "expected parent act-validation workflow not to install Markdown formatting tools" + ) + assert "docker info" in act_workflow, ( + "expected parent act-validation workflow to verify Docker before act tests" + ) + assert "ACT_GITHUB_TOKEN: ${{ github.token }}" in act_workflow, ( + "expected parent act-validation workflow to expose github.token only to " + "nested act tests" + ) + assert "make test WITH_ACT=1" in act_workflow, ( + "expected parent act-validation workflow to run parent tests with act enabled" + ) diff --git a/tests/test_template.py b/tests/test_template.py index c87cf92..a266f11 100644 --- a/tests/test_template.py +++ b/tests/test_template.py @@ -6,15 +6,6 @@ verify generated files, Make targets, package imports, and Rust-specific output without requiring callers to inspect the rendered project tree manually. -Typical usage is to run this module through pytest after changing template -files, generated Makefile targets, or package layout: - -Examples --------- -Run the generated-template checks directly:: - - python -m pytest tests/test_template.py -v - The tests create temporary projects, install generated dependencies through the rendered ``make all`` target, and may download toolchain packages into the normal user caches used by those generated projects. @@ -22,136 +13,30 @@ from __future__ import annotations -import shlex +import ast +import shutil +import subprocess from pathlib import Path import pytest -from pytest_copier.plugin import CopierFixture, CopierProject +from pytest_copier.plugin import CopierFixture from syrupy.assertion import SnapshotAssertion - -def run_quality_gates(project: CopierProject) -> None: - """Run the rendered project's public quality gate. - - Parameters - ---------- - project - Rendered ``pytest-copier`` project whose root contains the generated - Makefile. - - Returns - ------- - None - The helper delegates validation to the generated project's ``make all`` - target. - - Raises - ------ - AssertionError - Raised by ``pytest-copier`` if the command exits unsuccessfully. - - Examples - -------- - Validate a rendered project before making assertions about its files:: - - project = copier.copy(tmp_path / "pure", use_rust=False) - run_quality_gates(project) - """ - project.run("make all") - - -def check_generated_import(project: CopierProject, package: str, greeting: str) -> None: - """Import a generated package and assert its greeting. - - Parameters - ---------- - project - Rendered project whose managed environment should contain the package. - package - Import name to load through ``importlib.import_module``. - greeting - Expected return value from the generated package's ``hello`` function. - - Returns - ------- - None - The helper succeeds when the generated import and assertion succeed. - - Raises - ------ - AssertionError - Raised by ``pytest-copier`` if the command exits unsuccessfully. - - Examples - -------- - Check the generated pure-Python package import path:: - - check_generated_import(project, "pure_pkg", "hello from Python") - """ - script = ( - "import importlib; " - f"module = importlib.import_module({package!r}); " - f"assert module.hello() == {greeting!r}" - ) - project.run(f"uv run python -c {shlex.quote(script)}") - - -def read_generated_file(project: CopierProject, relative_path: str) -> str: - """Read a rendered project file as UTF-8 text. - - Parameters - ---------- - project - Rendered ``pytest-copier`` project to read from. - relative_path - Path to the generated file, relative to the rendered project root. - - Returns - ------- - str - File contents decoded as UTF-8. - - Raises - ------ - FileNotFoundError - Raised if the requested generated file does not exist. - - Examples - -------- - Read the generated Makefile for target assertions:: - - makefile = read_generated_file(project, "Makefile") - """ - return (project / relative_path).read_text(encoding="utf-8") - - -def assert_common_make_targets(makefile: str) -> None: - """Assert Makefile targets shared by all generated variants. - - Parameters - ---------- - makefile - UTF-8 text of a generated Makefile. - - Returns - ------- - None - The helper returns after all common target assertions pass. - - Raises - ------ - AssertionError - Raised when a required shared target or cleanup path is missing. - - Examples - -------- - Validate shared targets after reading a generated Makefile:: - - assert_common_make_targets(makefile) - """ - assert "lint-python: build" in makefile, "Makefile should expose lint-python" - assert "lint: lint-python" in makefile, "lint should delegate to lint-python" - assert ".uv-cache .uv-tools" in makefile, "clean should remove uv state dirs" +from tests.helpers.generated_files import ( + parse_toml_file, + read_generated_text, +) +from tests.helpers.rendering import ( + check_generated_import, + read_generated_file, + render_project, + run_quality_gates, +) +from tests.helpers.tooling_contracts import ( + assert_ci_coverage_action_contract, + assert_common_make_targets, + assert_generated_tooling_contracts, +) def test_python_only_help_output_snapshot( @@ -194,7 +79,12 @@ def test_python_only_help_output_snapshot( use_rust=False, ) - assert project.run("make help") == snapshot + help_output = project.run("make help") + for target in ["build", "check-fmt", "lint", "typecheck", "audit", "test", "help"]: + assert f" {target}" in help_output, ( + f"expected generated help output to list the {target!r} target" + ) + assert help_output == snapshot @pytest.mark.parametrize( @@ -244,7 +134,27 @@ def test_pure_module_snapshot( use_rust=use_rust, ) - assert read_generated_file(project, f"{package_name}/pure.py") == snapshot + pure_module = read_generated_file(project, f"{package_name}/pure.py") + parsed_module = ast.parse(pure_module) + hello_functions = [ + node + for node in parsed_module.body + if isinstance(node, ast.FunctionDef) and node.name == "hello" + ] + assert len(hello_functions) == 1, ( + "expected generated pure.py to define exactly one hello function" + ) + hello_function = hello_functions[0] + assert not hello_function.args.args, ( + "expected generated pure.py hello function to accept no positional arguments" + ) + assert isinstance(hello_function.returns, ast.Name), ( + "expected generated pure.py hello function to declare a return annotation" + ) + assert hello_function.returns.id == "str", ( + "expected generated pure.py hello function to return str" + ) + assert pure_module == snapshot def test_python_only_template(copier: CopierFixture, tmp_path: Path) -> None: @@ -274,20 +184,20 @@ def test_python_only_template(copier: CopierFixture, tmp_path: Path) -> None: ) run_quality_gates(proj) - assert not ( - proj / "rust_extension" - ).exists(), "rust_extension directory should not exist for Python-only template" - assert not ( - proj / "docs" / "rust-extension.md" - ).exists(), "Rust documentation should not be generated for Python-only template" - assert ( - "maturin" not in (proj / "pyproject.toml").read_text(encoding="utf-8") - ), "maturin should not be in pyproject.toml for Python-only template" + assert not (proj / "rust_extension").exists(), ( + "rust_extension directory should not exist for Python-only template" + ) + assert not (proj / "docs" / "rust-extension.md").exists(), ( + "Rust documentation should not be generated for Python-only template" + ) + assert "maturin" not in (proj / "pyproject.toml").read_text(encoding="utf-8"), ( + "maturin should not be in pyproject.toml for Python-only template" + ) makefile = read_generated_file(proj, "Makefile") assert_common_make_targets(makefile) - assert ( - "lint-rust" not in makefile - ), "Python-only Makefile should not expose lint-rust" + assert "lint-rust" not in makefile, ( + "Python-only Makefile should not expose lint-rust" + ) check_generated_import(proj, "pure_pkg", "hello from Python") @@ -322,27 +232,95 @@ def test_rust_template(copier: CopierFixture, tmp_path: Path) -> None: ) run_quality_gates(proj) - assert ( - proj / "rust_extension" - ).exists(), "rust_extension directory should exist for Rust template" - assert ( - proj / "docs" / "rust-extension.md" - ).exists(), "Rust documentation should be generated for Rust template" - assert ( - "maturin" in (proj / "pyproject.toml").read_text(encoding="utf-8") - ), "maturin should be in pyproject.toml for Rust template" + assert (proj / "rust_extension").exists(), ( + "rust_extension directory should exist for Rust template" + ) + assert (proj / "docs" / "rust-extension.md").exists(), ( + "Rust documentation should be generated for Rust template" + ) + assert "maturin" in (proj / "pyproject.toml").read_text(encoding="utf-8"), ( + "maturin should be in pyproject.toml for Rust template" + ) makefile = read_generated_file(proj, "Makefile") assert_common_make_targets(makefile) - assert ( - "lint-rust: build whitaker" in makefile - ), "Rust Makefile should expose lint-rust" - assert ( - "cargo is required for Rust tests" in makefile - ), "Rust Makefile should fail clearly when cargo is unavailable" + assert "lint-rust: build whitaker" in makefile, ( + "Rust Makefile should expose lint-rust" + ) + assert "cargo is required for Rust tests" in makefile, ( + "Rust Makefile should fail clearly when cargo is unavailable" + ) check_generated_import(proj, "rust_pkg", "hello from Rust") +def test_rust_template_make_test_runs_doctests( + copier: CopierFixture, tmp_path: Path +) -> None: + """Validate that Rust-enabled generated projects gate doctests. + + Parameters + ---------- + copier : CopierFixture + Fixture used to render the template into a temporary project. + tmp_path : Path + Temporary directory used as the generated project root. + + Returns + ------- + None + The test fails via assertions when the generated ``make test`` target + does not run Rust documentation tests. + + Raises + ------ + None + Expected failures are captured through pytest assertions. + + Notes + ----- + The test injects a deliberately broken Rust doctest and verifies that the + generated project's public ``make test`` target reports the doctest + failure. + """ + proj = copier.copy( + tmp_path / "rust-doctest", + project_name="RustDoctest", + package_name="rust_doctest_pkg", + use_rust=True, + ) + lib_rs = proj / "rust_extension" / "src" / "lib.rs" + lib_rs.write_text( + lib_rs.read_text(encoding="utf-8") + + """ + +/// Deliberately broken doctest used by the parent template regression test. +/// +/// ``` +/// let status = std::process::ExitCode::SUCCESS; +/// assert!(status.success()); +/// ``` +pub fn doctest_regression_marker() {} +""", + encoding="utf-8", + ) + make = shutil.which("make") + assert make is not None, "expected make to be available for generated tests" + + result = subprocess.run( + [make, "test"], + cwd=proj.path, + check=False, + capture_output=True, + text=True, + ) + + output = f"{result.stdout}\n{result.stderr}" + assert result.returncode != 0, "expected make test to fail on broken doctests" + assert "no method named `success`" in output, ( + "expected make test to compile doctests, exposing the broken example" + ) + + def test_rust_template_custom_package(copier: CopierFixture, tmp_path: Path) -> None: """Ensure templating uses the provided package name. @@ -373,12 +351,146 @@ def test_rust_template_custom_package(copier: CopierFixture, tmp_path: Path) -> ) run_quality_gates(proj) - assert ( - proj / "rust_extension" - ).exists(), "rust_extension directory should exist for custom package Rust template" + assert (proj / "rust_extension").exists(), ( + "rust_extension directory should exist for custom package Rust template" + ) text = (proj / "pyproject.toml").read_text(encoding="utf-8") - assert ( - "custom_pkg" in text - ), "custom package name should appear in pyproject.toml" + assert "custom_pkg" in text, "custom package name should appear in pyproject.toml" check_generated_import(proj, "custom_pkg", "hello from Rust") + + +@pytest.mark.parametrize( + ("target_dir", "project_name", "package_name", "use_rust"), + [ + ("tooling-pure", "ToolingPure", "tooling_pure", False), + ("tooling-rust", "ToolingRust", "tooling_rust", True), + ], +) +def test_generated_tooling_contracts( + copier: CopierFixture, + tmp_path: Path, + target_dir: str, + project_name: str, + package_name: str, + use_rust: bool, +) -> None: + """Generated variants expose the expected Python and optional Rust tooling. + + Parameters + ---------- + copier + ``pytest-copier`` fixture used to render the template. + tmp_path + Temporary directory where the rendered project is created. + target_dir + Temporary project directory name for the rendered variant. + project_name + Project name answer passed to Copier. + package_name + Package name answer passed to Copier. + use_rust + Whether the rendered variant includes the optional Rust extension. + + Returns + ------- + None + The test passes when the generated tooling contracts are satisfied. + """ + project = render_project( + tmp_path / target_dir, + copier, + project_name=project_name, + package_name=package_name, + use_rust=use_rust, + ) + + run_quality_gates(project) + project.run("uv tool run mbake validate Makefile") + + pyproject = parse_toml_file(project / "pyproject.toml") + agents = read_generated_text(project / "AGENTS.md") + makefile = read_generated_text(project / "Makefile") + ci_workflow = read_generated_text(project / ".github" / "workflows" / "ci.yml") + act_validation_workflow = read_generated_text( + project / ".github" / "workflows" / "act-validation.yml" + ) + release_workflow = read_generated_text( + project / ".github" / "workflows" / "release.yml" + ) + build_wheels_workflow = read_generated_text( + project / ".github" / "workflows" / "build-wheels.yml" + ) + build_wheels_action = read_generated_text( + project / ".github" / "actions" / "build-wheels" / "action.yml" + ) + pure_wheel_action = read_generated_text( + project / ".github" / "actions" / "pure-python-wheel" / "action.yml" + ) + + assert_generated_tooling_contracts( + package_name=package_name, + agents=agents, + pyproject=pyproject, + makefile=makefile, + ci_workflow=ci_workflow, + act_validation_workflow=act_validation_workflow, + release_workflow=release_workflow, + build_wheels_workflow=build_wheels_workflow, + build_wheels_action=build_wheels_action, + pure_wheel_action=pure_wheel_action, + use_rust=use_rust, + ) + + +@pytest.mark.parametrize( + ("target_dir", "project_name", "package_name", "use_rust"), + [ + ("workflow-pure", "WorkflowPure", "workflow_pure", False), + ("workflow-rust", "WorkflowRust", "workflow_rust", True), + ], +) +def test_generated_github_workflows_match_act_validation_contract( + copier: CopierFixture, + tmp_path: Path, + target_dir: str, + project_name: str, + package_name: str, + use_rust: bool, +) -> None: + """Rendered workflows expose stable black-box inputs for act validation. + + Parameters + ---------- + copier + ``pytest-copier`` fixture used to render the template. + tmp_path + Temporary directory where the rendered project is created. + target_dir + Temporary project directory name for the rendered variant. + project_name + Project name answer passed to Copier. + package_name + Package name answer passed to Copier. + use_rust + Whether the rendered variant includes the optional Rust extension. + + Returns + ------- + None + The test passes when the generated CI coverage action contract matches + the act validation expectations. + """ + project = render_project( + tmp_path / target_dir, + copier, + project_name=project_name, + package_name=package_name, + use_rust=use_rust, + ) + ci_workflow = read_generated_text(project / ".github" / "workflows" / "ci.yml") + assert_ci_coverage_action_contract( + ci_workflow=ci_workflow, + package_name=package_name, + use_rust=use_rust, + ) diff --git a/tests/utilities.py b/tests/utilities.py index f854df4..29a7fc7 100644 --- a/tests/utilities.py +++ b/tests/utilities.py @@ -15,7 +15,9 @@ ----------- Only canonical local Unix socket paths are accepted. Remote Docker endpoints, malformed ``DOCKER_HOST`` values, missing sockets, and sockets outside the -allowed runtime directories are removed or ignored. +allowed runtime directories are removed or ignored. Host GitHub authentication +tokens are removed so stale local credentials cannot break public action clones +inside ``act``. Examples -------- @@ -31,6 +33,8 @@ from pathlib import Path from urllib.parse import urlparse +GITHUB_AUTH_ENV_VARS = ("GITHUB_TOKEN", "GH_TOKEN") + def _resolved_socket_from_docker_host( docker_host: str, allowed_dirs: tuple[Path, ...] @@ -150,6 +154,8 @@ def docker_environment() -> dict[str, str]: subprocess.run(command, env=docker_environment(), check=False) """ env = os.environ.copy() + for variable in GITHUB_AUTH_ENV_VARS: + env.pop(variable, None) docker_host = env.get("DOCKER_HOST") if docker_host is not None: socket_path = _resolved_socket_from_docker_host(