diff --git a/.github/actions/check-quality/action.yml b/.github/actions/check-quality/action.yml index 473c35132..f22834b02 100644 --- a/.github/actions/check-quality/action.yml +++ b/.github/actions/check-quality/action.yml @@ -73,6 +73,10 @@ runs: shell: bash run: python scripts/check_markdown_links.py + - name: Check test layout + shell: bash + run: python scripts/check_test_layout.py + - name: Test Specula CI wiring shell: bash working-directory: .github/specula diff --git a/.github/actions/validate-nvx/action.yml b/.github/actions/validate-nvx/action.yml index 7e693dfb3..e824019ec 100644 --- a/.github/actions/validate-nvx/action.yml +++ b/.github/actions/validate-nvx/action.yml @@ -11,7 +11,7 @@ runs: set -euo pipefail python3 -m pip install --requirement scripts/setup/azure/requirements.txt python3 -m compileall -q scripts - python3 -m unittest scripts/test_performance.py scripts/test_nvx_tools.py scripts/test_guest_binaries.py scripts/test_microvm_tests.py scripts/test_nightly_ci.py scripts/test_development_release.py scripts/test_crate_release.py scripts/test_host_telemetry.py scripts/test_time_abi.py scripts/test_tool_versions.py scripts/test_markdown_links.py scripts/test_module_diagrams.py scripts/test_nvx_rs_mutants.py scripts/test_setup_azure.py scripts/test_setup_azure_linux.py scripts/test_setup_azure_windows.py scripts/test_powershell_testing.py -v + python3 -m unittest scripts/test_performance.py scripts/test_nvx_tools.py scripts/test_guest_binaries.py scripts/test_microvm_tests.py scripts/test_nightly_ci.py scripts/test_development_release.py scripts/test_crate_release.py scripts/test_host_telemetry.py scripts/test_time_abi.py scripts/test_tool_versions.py scripts/test_markdown_links.py scripts/test_module_diagrams.py scripts/test_nvx_rs_mutants.py scripts/test_setup_azure.py scripts/test_setup_azure_linux.py scripts/test_setup_azure_windows.py scripts/test_powershell_testing.py scripts/test_test_layout.py scripts/test_test_changes.py -v python3 scripts/test_adversarial.py -v python3 .github/skills/nvx-host-connect/scripts/test_hosts.py -v python3 scripts/nvx.py --help @@ -32,7 +32,7 @@ runs: if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } python -m compileall -q scripts if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - python -m unittest scripts/test_performance.py scripts/test_nvx_tools.py scripts/test_guest_binaries.py scripts/test_microvm_tests.py scripts/test_nightly_ci.py scripts/test_development_release.py scripts/test_crate_release.py scripts/test_host_telemetry.py scripts/test_time_abi.py scripts/test_tool_versions.py scripts/test_markdown_links.py scripts/test_module_diagrams.py scripts/test_nvx_rs_mutants.py scripts/test_setup_azure.py scripts/test_setup_azure_linux.py scripts/test_setup_azure_windows.py scripts/test_powershell_testing.py -v + python -m unittest scripts/test_performance.py scripts/test_nvx_tools.py scripts/test_guest_binaries.py scripts/test_microvm_tests.py scripts/test_nightly_ci.py scripts/test_development_release.py scripts/test_crate_release.py scripts/test_host_telemetry.py scripts/test_time_abi.py scripts/test_tool_versions.py scripts/test_markdown_links.py scripts/test_module_diagrams.py scripts/test_nvx_rs_mutants.py scripts/test_setup_azure.py scripts/test_setup_azure_linux.py scripts/test_setup_azure_windows.py scripts/test_powershell_testing.py scripts/test_test_layout.py scripts/test_test_changes.py -v if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } python scripts\test_adversarial.py -v if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 4ac376ee7..db1ef3deb 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -33,6 +33,19 @@ nested repository. Do not claim unavailable hardware or cross-platform gates passed; report the missing prerequisite and rely on CI when appropriate. +## Tests + +- Put a Rust module's unit tests in its `tests` module, in files of their own + (`src/a/b.rs` declares `#[cfg(test)] mod tests;` and keeps them in + `src/a/b/tests.rs`), never inline or as test-only items in production files. + Python unit tests are `test_*.py` modules in `scripts/`. Run + `python3 scripts/check_test_layout.py`; a test hook that cannot move needs an + entry in `scripts/test-layout-allowlist.json`, and moving test code out needs + `--tighten`. See [Test layout](../doc/contribute.md#test-layout). +- A change to an existing test, its data, or its CI wiring gets the + `tests-modified` label; explain each such change in the pull request. Run + `python3 scripts/check_test_changes.py --base origin/dev` to list them first. + ## Documentation - In [`nvx-rs/README.md`](../nvx-rs/README.md#layout), each row of the module diff --git a/.github/workflows/test-changes.yml b/.github/workflows/test-changes.yml new file mode 100644 index 000000000..513ab1fea --- /dev/null +++ b/.github/workflows/test-changes.yml @@ -0,0 +1,80 @@ +name: Test changes + +# Flags a pull request that modifies or removes existing tests, test data, +# end-to-end scenarios, test doubles, test policy, or CI test wiring: it +# labels the pull request tests-modified and keeps one comment that lists the +# changes (doc/ci.md#test-changes). pull_request_target runs the workflow as +# the default branch holds it, with a token that may label and comment, for +# pull requests from forks too. So the job runs the default branch's script, +# which reads the pull request's commits as data and never runs their code. +on: + pull_request_target: + types: + - opened + - synchronize + - reopened + - ready_for_review + - edited + +permissions: {} + +jobs: + flag: + name: Flag changes to existing tests + # An edit changes what the job reads only when it changes the base branch. + if: github.event.action != 'edited' || github.event.changes.base != null + runs-on: ubuntu-latest + timeout-minutes: 10 + # In the job, so that a run whose job the condition skips, as an edit of + # the title does, cancels no comparison. + concurrency: + group: test-changes-${{ github.event.pull_request.number }} + cancel-in-progress: true + permissions: + contents: read + pull-requests: write + steps: + # The default branch's script runs, whatever the pull request's base: + # pull_request_target runs the default branch's workflow, and the + # checkout names that branch, so that no other branch's script runs + # with the token. It fetches every branch, the base among them. + - name: Checkout + uses: actions/checkout@v5 + with: + ref: ${{ github.event.repository.default_branch }} + fetch-depth: 0 + persist-credentials: false + + # The pull request's commits are data: nothing of them runs. + - name: Fetch the pull request's commits + id: fetch + env: + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + NUMBER: ${{ github.event.pull_request.number }} + run: | + set -euo pipefail + git fetch --no-tags --quiet origin \ + "+refs/pull/${NUMBER}/head:refs/remotes/pull/head" + # A push after the event that started this run starts another, which + # flags the newer head. + if [[ "$(git rev-parse refs/remotes/pull/head)" == "${HEAD_SHA}" ]]; then + echo "current=true" >> "${GITHUB_OUTPUT}" + else + echo "current=false" >> "${GITHUB_OUTPUT}" + echo "The pull request's head moved on; a newer run flags it." + fi + + - name: Flag changes to existing tests + if: steps.fetch.outputs.current == 'true' + env: + GH_TOKEN: ${{ github.token }} + BASE_REF: ${{ github.event.pull_request.base.ref }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + NUMBER: ${{ github.event.pull_request.number }} + run: >- + python3 scripts/check_test_changes.py + --base "origin/${BASE_REF}" + --head "${HEAD_SHA}" + --repository "${GITHUB_REPOSITORY}" + --summary "${GITHUB_STEP_SUMMARY}" + --publish "${NUMBER}" diff --git a/doc/ci.md b/doc/ci.md index c0f0d3f34..87cc0b486 100644 --- a/doc/ci.md +++ b/doc/ci.md @@ -1811,7 +1811,9 @@ start the workflow, and a manually dispatched run notifies the person who started it. The job checks out the default branch, never the run's code. Its token can read the repository, the run's jobs and logs, and their check annotations, -and write issues, and no other workflow can write issues. The job runs one +and write issues, and no other workflow can write issues; the +[Test changes](#test-changes) workflow labels and comments on pull requests +with a token that writes pull requests instead. The job runs one report at a time; a newer reporter can replace one that waits for its turn, but it reports the latest nightly result anyway. Its job summary repeats the report. @@ -1866,6 +1868,44 @@ owner triages a new report within a working day: To hand triage over or to rotate it, change the workflow's `--assignee` options, and reassign the open report. +## Test changes + +The [`test-changes.yml`](../.github/workflows/test-changes.yml) workflow flags +a pull request that modifies or removes an existing test asset, as +[Changes to existing tests](contribute.md#changes-to-existing-tests) +describes: it labels the pull request `tests-modified`, which the workflow +creates when the repository lacks it, and keeps one comment that lists the +changes, which a hidden marker at its start identifies. It runs when a pull +request to any branch, a stacked one included, is opened, reopened, pushed +to, marked ready for review, or retargeted, drafts included, and replaces a +run of the same pull request in progress. A run that finds no change to an +existing test removes the label and rewrites the comment, if there is one, +to say so; the label then blocks nothing, and `Required status check` does +not wait for the workflow. Its job summary lists every change, where the +comment, which GitHub limits to 65,536 characters, may cut a section short. + +`pull_request_target` gives the job a token that can label and comment, for +pull requests from forks too, so the job never runs a pull request's code. It +runs the workflow and script of the default branch: `pull_request_target` +always runs that branch's workflow, and its checkout names that branch, with +every other fetched and without the token's credentials kept. It fetches the +pull request's commits as data, and +[`check_test_changes.py`](../scripts/check_test_changes.py) reads their files +from Git's objects: it parses Rust and Python without running them and +compares the merge base of the pull request's base branch and head with the +head. A head that moved on since the event exits, since the push starts +another run. The token can read the repository's contents and write pull +requests, with which GitHub labels and comments on them, but not issues, which +only the [nightly failure report](#nightly-failure-reports) writes. A change +to the workflow or to the script takes effect once it merges into `dev`, and +the script itself counts as test policy, so a pull request that changes it is +flagged once it does. + +The `quality` job checks the [test layout](contribute.md#test-layout) with +[`check_test_layout.py`](../scripts/check_test_layout.py), whose rules say +where tests live, which test hooks a production file may hold, and which the +flagging script shares to read Rust. + ## crates.io releases A scheduled or manually dispatched full run of `dev` publishes the diff --git a/doc/contribute.md b/doc/contribute.md index af400caff..d0639c56e 100644 --- a/doc/contribute.md +++ b/doc/contribute.md @@ -3,7 +3,8 @@ NVX takes contributions as pull requests to the `dev` branch of [microsoft/nvx](https://github.com/microsoft/nvx). This guide covers the workflow, the conventions that reviewers expect, the quality gates, test -ownership, the expectations of tests, and how OpenVMM changes reach NVX. +ownership, the layout and expectations of tests, and how OpenVMM changes +reach NVX. Report security vulnerabilities as [SECURITY.md](../SECURITY.md) describes, never in a public issue. @@ -84,6 +85,7 @@ the Python, shell, and link checks. | Guest and setup shell scripts | ShellCheck, and shfmt with four-space indentation and indented `case` branches. Scripts are POSIX `sh` unless check-quality checks them as Bash. | [check-quality](../.github/actions/check-quality/action.yml) | | `scripts/setup/setup-windows-whp.ps1` | PowerShell syntax | [check-quality](../.github/actions/check-quality/action.yml) | | Markdown files and `doc/` paths | Every relative link and image in a tracked Markdown file resolves to a tracked file, and every anchor to a heading slug that GitHub generates, ignoring code. Every `doc/` path that a tracked file mentions resolves too, except the test fixtures that `DOC_PATH_FIXTURES` lists. | [check-quality](../.github/actions/check-quality/action.yml), [check_markdown_links.py](../scripts/check_markdown_links.py) | +| Tests | Every test lives in a test file, and production files hold only the test hooks that the allowlist allows; see [Test layout](#test-layout) | [check-quality](../.github/actions/check-quality/action.yml), [check_test_layout.py](../scripts/check_test_layout.py) | | `nvx-rs/` | rustfmt, Clippy and rustdoc with warnings denied, tests including a bundled-release fixture, the minimum supported Rust version, a macOS check, and, through `nvx.py`, the musl lint, tests, build, digest manifest, and static-ELF check of the guest binaries; see [Rust crates](ci.md#rust-crates) | [check-nvx-rs](../.github/actions/check-nvx-rs/action.yml) | | `tests/runtime-adapter/` | rustfmt, then Clippy and rustdoc with warnings denied, the build, the tests, and the minimum supported Rust version of `nvx`, in the protocol-only, host-only, and, on Linux, static musl guest-only configurations; see [Rust crates](ci.md#rust-crates) | [check-nvx-rs](../.github/actions/check-nvx-rs/action.yml) | | NVX CLI | Compiles `scripts/`, runs every `scripts/test_*.py` module, and checks command help, on Linux and Windows, and compares the copies of OpenVMM's protocol vectors in `nvx-rs/tests/vectors/openvmm/`, and of the identities of its built-in CPU profiles in `nvx-rs/src/host/cli/cpu_generations.json`, with the submodule's files | [validate-nvx](../.github/actions/validate-nvx/action.yml), [ci.yml](../.github/workflows/ci.yml) | @@ -125,9 +127,10 @@ scenarios drive them as commands, compile egress policies with Python client of the control protocol, lifecycle, or egress logic. Unit tests for the host tooling live in `scripts/test_*.py`. The `nvx` crate -keeps unit tests alongside its Rust source and integration tests in its -`tests` directory, and also runs `tests/guest_tools.rs` against the exact -guest-profile binaries. Tests of the NVX +keeps the unit tests of each module in the module's `tests` module, in files +of their own beside its source, as [Test layout](#test-layout) describes, and +integration tests in its `tests` directory, and also runs +`tests/guest_tools.rs` against the exact guest-profile binaries. Tests of the NVX mechanics, such as the conformance vectors of the control protocol, belong in `nvx`. Tests of what an independent adapter observes through the public API of `nvx` alone, such as its own framing over the control @@ -137,6 +140,74 @@ suite the microVM jobs run on each backend with `scripts/nvx.py test-runtime-adapter --backend BACKEND`. [Continuous integration](ci.md) describes how CI checks OpenVMM's own CI and what the NVX microVM suite covers. +## Test layout + +Tests live apart from the code that they test, so that a change to one never +hides in a change to the other, and so that the guest images, which carry the +code of `nvx-rs` but not its tests, never rebuild for a test. A Rust crate's +test files are those beneath its `tests` directory and, in its `src/`, the +`tests.rs` files and the files beneath a `tests` directory. Its other files in +`src/` are production files, which declare their unit tests only as an +out-of-line `tests` module that only tests compile: + +```rust +#[cfg(test)] +mod tests; +``` + +`src/a/b.rs` keeps its unit tests in `src/a/b/tests.rs`, or in +`src/a/b/tests/mod.rs` and the files beside it, and `src/a/mod.rs` keeps them +in `src/a/tests.rs`. A test file belongs to a module that the crate's library, +one of its binaries, or one of its integration tests compiles for tests, +through the `mod` declarations from the target's root, since no other test +runs. `Cargo.toml` names the targets, or Cargo discovers them, and a target +with `test = false` runs none; a declaration that a cfg excludes from every +build of tests, such as `#[cfg(any())]` or `#[cfg(not(test))]`, compiles +nothing for them. The Rust files of +`tests/fixtures/` and `tests/vectors/` are data, not modules. The `tests` +module may be `pub(crate)`, so that the tests of other modules use its +fixtures and helpers, and its cfg may add conditions, as +`#[cfg(all(test, unix))]` does. The code that only tests use belongs in it +too: fixtures, helpers, test doubles, and the methods of production types that +only tests call, in an `impl` block of the `tests` module. A production file +holds no other test code: outside its function bodies, no inline module, no +module of another name, and no item that a cfg requiring `test` guards; no +test attribute, such as `#[test]`, `#[should_panic]`, or `#[ignore]`; no +`proptest!` block; and no `#![cfg(test)]`. What remains are test hooks, the +code that cannot move out of the code that it hooks: the attributes and +`cfg!` invocations whose predicates mention `test`, such as a field, a +statement, or a match arm that only tests compile, or a +`#[cfg_attr(test, derive(...))]`. The seams of +[`scripts/test-layout-allowlist.json`](../scripts/test-layout-allowlist.json) +list how many each production file may hold. A change that adds a hook +records it there, and a reviewer checks that the hook cannot move. + +The production files that predate the layout keep their test code until it +moves out: the allowlist's baseline lists each with the numbers of its test +hooks and of its tests, which may only go down. So a new test of such a file +goes in its `tests` module, to which its other tests move first. Once a file's +tests have moved, `--tighten` lowers the allowlist to what the files hold and +moves the file from the baseline to the seams; it never raises an entry or +adds one. + +Python unit tests are the `test_*.py` modules of `scripts/`, +`.github/specula/tests/`, and `.github/skills/*/scripts/`, and the end-to-end +scenario drivers are the `*_tests.py` modules of `scripts/nvx_tools/`, whose +guest workloads live in `scripts/nvx_tools/microvm_test_scripts/`. No other +module imports `unittest`, `pytest`, `doctest`, a `test_*` module, or the +PowerShell test support of `scripts/powershell_testing.py`. + +`check_test_layout.py` checks these rules, which check-quality runs. It reads +the package and target tables of each `Cargo.toml`, whose keys it reads when +they hold a string or a boolean on their own lines, and reports a declaration +of a target that it cannot read. It reads the files that Git tracks, so stage +a new file before you run it: + +```bash +python3 scripts/check_test_layout.py +python3 scripts/check_test_layout.py --tighten # after moving test code out +``` + ## Test expectations Tests take their expectations from what NVX specifies, never from output that @@ -169,6 +240,45 @@ crate's [Testing](../nvx-rs/README.md#testing) section describes. compare the mutants that each selection catches with [nvx-rs mutation testing](ci.md#nvx-rs-mutation-testing). +### Changes to existing tests + +A change to an existing test can hide the regression that the test caught, +so the [Test changes](ci.md#test-changes) workflow labels every pull request +that modifies or removes an existing test asset `tests-modified`, and keeps +one comment on it that lists each, with links to it before and after the +change. It compares the pull request's head with its merge base, item by +item: a test, a test helper, the imports and inner attributes of a test +module, and the test hooks of a production item; a fixture, a conformance +vector, a scenario list, or another data file of the tests; an end-to-end +scenario driver, guest workload, or guest probe; the fake OpenVMM; an +allowlist entry that is new or allows more, or a change to the layout check, +the flagging script, or their workflow; and a removed or changed line of the +actions and workflow that run the tests and of the `ci.yml` jobs that run +them, or an added line there that may skip them, and a test job that +`ci.yml` no longer has. New tests flag nothing, and neither does code that +moves unchanged under the same cfg predicates, its own and those that its +module inherits, such as a `tests` module that moves to its own file, nor +formatting, comments, lint attributes, visibility, or imports that a module +only adds. A test whose module no target compiles any more, or compiles under +other predicates, as a changed declaration of an ancestor module, or a +manifest that sets the `test` key of its target to false or roots the target +elsewhere, may make it, counts as removed or modified. Each target that +compiles a module, through each chain of declarations, counts, so a test that +one of them no longer compiles counts as modified. A comma counts in the +token tree of a macro, unless the macro is one of the standard library's that +take expressions and a trailing comma, such as `assert_eq!`. + +The label blocks nothing. It asks the reviewers to check each listed change +against the specification that the test derives from: that a modified +assertion still checks what it did, or that the behavior changed on purpose, +and that a removed test's coverage moved elsewhere or is no longer needed. +Explain each such change in the pull request's body. Run the comparison +before you push: + +```bash +python3 scripts/check_test_changes.py --base origin/dev +``` + ## OpenVMM ### Access diff --git a/doc/project-structure.md b/doc/project-structure.md index d7e4b946c..0341920b3 100644 --- a/doc/project-structure.md +++ b/doc/project-structure.md @@ -8,7 +8,7 @@ are build products or caches and are not part of the tracked source tree. The | Path | Purpose | | --- | --- | -| `.github/workflows` | CI, nightly failure report, module diagram, release verification, adversarial campaign, and Copilot environment workflows | +| `.github/workflows` | CI, nightly failure report, test change flag, module diagram, release verification, adversarial campaign, and Copilot environment workflows | | `.github/actions` | Reusable local CI actions | | `.github/skills` | Copilot agent skills for common development workflows | | `.github/agents` | Copilot custom agents: the bounded adversarial strategist and the issue fixer | @@ -52,7 +52,8 @@ nvx/ | | |-- package-platform.yml Reusable platform release package job | | |-- run-nvx-microvm-tests.yml Reusable NVX microVM test job | | |-- run-platform.yml Reusable platform benchmark job -| | `-- specula-release.yml Specula verification of published releases +| | |-- specula-release.yml Specula verification of published releases +| | `-- test-changes.yml Label and comment for pull requests that change existing tests | |-- actionlint.yaml Self-hosted runner labels for actionlint | `-- copilot-instructions.md Repository-wide Copilot instructions |-- .vscode/settings.json Shared VS Code settings @@ -157,6 +158,8 @@ nvx/ | | |-- setup-windows-whp.ps1 Windows/WHP development-host and Actions runner bootstrap | | `-- tool-versions.conf Shared tool release and checksum pins | |-- check_markdown_links.py Markdown link, anchor, and doc/ path check run by CI +| |-- check_test_changes.py Flags the changes of a pull request to existing tests +| |-- check_test_layout.py Check that tests live apart from the code that they test | |-- module_diagrams.py Module diagram generator of the nvx crate | |-- nvx-hosts.example.json Template for the ignored .nvx-hosts.json | |-- nvx.py Supported command-line entry point @@ -164,6 +167,7 @@ nvx/ | |-- powershell_testing.py PowerShell launches of the unit tests, under a shared default time limit | |-- publish_development_release.py Development release publisher run by CI | |-- publish_nvx_crate.py nvx crate publisher run by CI's full runs of dev +| |-- test-layout-allowlist.json The test hooks and tests that production files may hold | `-- test_*.py Python tooling unit tests |-- tests/ Tests that build on the repository's crates from outside them | `-- runtime-adapter/ Rust crate `runtime-adapter`, an independent adapter of nvx @@ -198,7 +202,8 @@ reusable workflows: `build-openvmm-binary.yml` builds OpenVMM for one target, `run-nvx-microvm-tests.yml` runs the NVX microVM tests on one backend, and `run-platform.yml` benchmarks one platform. The full runs of `ci.yml` on `dev` also publish the `nvx` crate to -crates.io. The other workflows report failed nightly CI runs, draw the module +crates.io. The other workflows report failed nightly CI runs, flag the pull +requests that change existing tests, draw the module diagrams of the `nvx` crate after merges, run adversarial campaigns, run the `nvx` crate's mutation tests, verify published releases with Specula, and prepare the @@ -330,7 +335,12 @@ the storage events of a failed Windows benchmark job, as beside `nvx.py` are the tooling's unit tests. They launch PowerShell through `powershell_testing.py`, which gives each launch a shared default time limit and reports what a launch that outlasts its limit printed, at once on stderr -and in the test's failure. +and in the test's failure. `check_test_layout.py` checks that every test of the +repository lives in a test file, and that production files hold only the test +hooks that `test-layout-allowlist.json` allows them, and +`check_test_changes.py` flags the pull requests that change existing tests; see +[Test layout](contribute.md#test-layout) and +[Changes to existing tests](contribute.md#changes-to-existing-tests). The `setup/` scripts bootstrap development hosts and GitHub Actions runners using the [tool manifest](../scripts/setup/tool-versions.conf); diff --git a/doc/setup.md b/doc/setup.md index e3c8d718d..ffe0ea68a 100644 --- a/doc/setup.md +++ b/doc/setup.md @@ -243,10 +243,12 @@ shfmt -d -ln posix -i 4 -ci \ shfmt -d -ln bash -i 4 -ci \ .github/specula/setup-runner.sh guest/ubuntu/nvx-bashrc python3 scripts/check_markdown_links.py +python3 scripts/check_test_layout.py ``` Pyright runs in strict mode for both Linux and Windows platform APIs. The link -check reads the files that Git tracks, so stage a new file before you run it. +and test layout checks read the files that Git tracks, so stage a new file +before you run them. ### Adversarial campaign controller diff --git a/nvx-rs/README.md b/nvx-rs/README.md index 4cbeee4b9..95c7462c7 100644 --- a/nvx-rs/README.md +++ b/nvx-rs/README.md @@ -2658,6 +2658,14 @@ cd nvx-rs NVX_BUNDLE=skip cargo test --all-features # unit, conformance-vector, property, and fake-OpenVMM tests ``` +A module's unit tests belong in its `tests` module, out of line, in the +module's `tests.rs` file or in the files of its `tests` directory, which only +tests compile, with its fixtures and test helpers. Production code holds no +tests, only the test hooks that +[`test-layout-allowlist.json`](../scripts/test-layout-allowlist.json) allows, +but for the files that predate the layout, as +[Test layout](../doc/contribute.md#test-layout) describes. + The tests need no hypervisor. The fake-OpenVMM test drives `Instance` end to end against `nvx-fake-openvmm`, and the template-capture test runs templates' captures through the supervisor against it, up to a crash of the adapter whose diff --git a/scripts/check_test_changes.py b/scripts/check_test_changes.py new file mode 100644 index 000000000..827dfd5d8 --- /dev/null +++ b/scripts/check_test_changes.py @@ -0,0 +1,2801 @@ +#!/usr/bin/env python3 +"""Flag the changes of a pull request to its repository's existing tests. + +Compares the head of a pull request with its merge base, and reports every +existing test asset that the head modifies or removes, in six categories: + +- test code: the items of the Rust test files and of the code that only tests + compile in production files, such as an inline `tests` module, a test + helper, or a test hook; the functions, classes, and other statements of the + Python unit tests and of their PowerShell test support; +- test data: fixtures, conformance vectors, and other data files of the tests, + and the scenario and adversarial case lists; +- end-to-end tests: the scenario drivers, the `*_tests.py` modules of + scripts/nvx_tools/, the guest scripts of the microVM tests, and the guest + probes; +- test doubles: the fake OpenVMM of `nvx::testing` and its binary; +- test policy: an allowlist entry of scripts/test-layout-allowlist.json that + is new or allows more, and any change to the layout check, to this script, + or to its workflow; +- CI test wiring: a removed or changed line of the actions and workflow that + run the tests, and of the ci.yml jobs that run them, or an added line there + that may skip them, such as an `if:`, a `continue-on-error`, or a + conditional that wraps a test command; a test job + that ci.yml no longer has, and a test action, workflow, command, module, or + scenario that it no longer names; and a scenario that leaves a backend's + shards. Comments count for none of these. + +It compares items, not lines: a Rust item is the same item where it keeps its +module path, kind, name, and cfg, so an inline `tests` module that moves to +its own file keeps its items. Formatting, comments, lint attributes, imports' +order, and visibility do not count, nor does an item that moves unchanged +under the same cfg predicates, its own and those that it inherits from its +module and file, in each way that a target compiles it, such as a test helper +that moves into a `tests` module with its cfg(test) dropped. A new test flags +nothing, and a test whose module no +target of its crate compiles any more, as when its crate's manifest sets the +`test` key of its target to false or roots the target elsewhere, counts as +removed. + +The script reads the commits' blobs and never runs their code, so a +workflow that pull_request_target triggers can run it, from the default +branch, on the commits of any pull request. +""" + +from __future__ import annotations + +import argparse +import ast +import copy +import difflib +import json +import os +import posixpath +import re +import subprocess +import sys +import urllib.parse +from collections import Counter, defaultdict +from collections.abc import Callable, Collection, Iterable, Mapping, Sequence +from dataclasses import dataclass, field +from pathlib import Path +from typing import cast + +import check_test_layout as layout +from check_test_layout import IDENT, LITERAL, PUNCT, Item, RustFile, Token + +REPO_ROOT = layout.REPO_ROOT +DEFAULT_BASE = "origin/dev" +LABEL = "tests-modified" +LABEL_COLOR = "d93f0b" +LABEL_DESCRIPTION = "Changes existing tests, test data, or test wiring" +COMMENT_MARKER = "" +# GitHub refuses comments longer than 65536 characters. +COMMENT_LIMIT = 60000 +ROWS_PER_SECTION = 50 + +TEST_CODE = "test code" +TEST_DATA = "test data" +END_TO_END = "end-to-end tests" +TEST_DOUBLES = "test doubles" +TEST_POLICY = "test policy" +CI_WIRING = "CI test wiring" +CATEGORIES = (TEST_CODE, TEST_DATA, END_TO_END, TEST_DOUBLES, TEST_POLICY, CI_WIRING) + +# The actions that flag a pull request. +FLAGGING = frozenset({"modified", "removed", "loosened", "added a line that may skip"}) + +POLICY_FILES = frozenset( + { + "scripts/check_test_changes.py", + "scripts/check_test_layout.py", + ".github/workflows/test-changes.yml", + } +) +# The actions and workflow that run tests and nothing else, whose every line is +# test wiring, and the workflow that names them, whose test actions, +# workflows, commands, modules, and scenarios are. +WIRING_FILES = frozenset( + { + ".github/actions/check-nvx-rs/action.yml", + ".github/actions/check-quality/action.yml", + ".github/actions/validate-nvx/action.yml", + ".github/workflows/run-nvx-microvm-tests.yml", + } +) +WIRING_TOKEN_FILES = frozenset({".github/workflows/ci.yml"}) +SHARDS_FILE = "scripts/nvx_tools/microvm_shards.json" +DOUBLES = re.compile(r"nvx-rs/src/testing/.+\.rs|nvx-rs/src/bin/nvx-fake-openvmm\.rs") +END_TO_END_FILES = re.compile( + r"scripts/nvx_tools/microvm_test_scripts/.+" + r"|guest/common/nvx-(?:identity-probe|sandbox-smoke|virtio-restore-probe)" +) +TEST_DATA_FILES = re.compile( + r"scripts/nvx_tools/adversarial_cases/.+|tests/runtime-adapter/scenarios\.txt" +) +# Lines that may keep tests from running when a test action or workflow adds +# them: a step's condition or its tolerance of failure, a command that ends the +# script early or skips tests, and a shell or PowerShell conditional or loop, +# which may wrap a test command without changing its line. +SKIPPING = re.compile( + r"continue-on-error|(?:^|\s)if:|\|\|\s*(?:true|:)(?:\s|$)|\bexit\s+0\b" + r"|--skip\b|--exclude\b" + r"|(?:^\s*|[;&|:]\s*)(?i:if|elif|elseif|else|then|fi|case|esac|switch|while" + r"|until|for|foreach|do|done|return)\b(?!:)" +) +# A Python test module that a test command names, such as scripts/test_a.py. +TEST_MODULE = re.compile(r"(?:\.[/\\])?(?:[\w.-]+[/\\])*test_\w+\.py") +WIRING_TOKENS = ( + re.compile( + r"uses:\s*(\./\.github/(?:actions/(?:check-nvx-rs|check-quality|validate-nvx)" + r"|workflows/run-nvx-microvm-tests\.yml))" + ), + re.compile(r"\bnvx\.py\s+((?:test|check)-[\w-]+)"), + re.compile(r"--scenario\s+([\w.-]+)"), + re.compile(r"scripts[/\\](test_\w+\.py)"), + re.compile(r"(\bcargo\b[^\n]*\b(?:test|nextest)\b[^\n]*)"), +) +LINT_ATTRIBUTES = frozenset({"allow", "deny", "expect", "forbid", "warn"}) + + +class FlagError(RuntimeError): + """Raised when the commits cannot be compared or the pull request cannot + be flagged.""" + + +# What changed. + + +@dataclass(frozen=True) +class Location: + path: str + line: int + + +@dataclass(frozen=True) +class Change: + """A change to a test asset: modified, removed, relocated, added, or one + of the actions of the test policy and CI wiring.""" + + category: str + action: str + kind: str + what: str + base: Location | None = None + head: Location | None = None + detail: str = "" + + @property + def flags(self) -> bool: + return self.action in FLAGGING + + def order(self) -> tuple[int, str, str, str, int, str, int, str]: + """Sorts changes by category, action, kind, and place.""" + base = self.base or Location("", 0) + head = self.head or Location("", 0) + return ( + CATEGORIES.index(self.category), + self.action, + self.kind, + base.path, + base.line, + head.path, + head.line, + self.what, + ) + + +# The ways that code compiles: for each target that compiles it, through each +# chain of declarations, the cfg predicates that must all hold. The +# conditions of code that compiles where nothing needs to hold, or nowhere, +# are ANYWHERE and empty. +Conditions = frozenset[frozenset[str]] +ANYWHERE: Conditions = frozenset({frozenset()}) + + +def _reduced(ways: Iterable[frozenset[str]]) -> Conditions: + """Returns ways without those that another, which needs fewer predicates + to hold, makes moot.""" + unique = set(ways) + return frozenset(way for way in unique if not any(other < way for other in unique)) + + +def _within(conditions: Conditions, predicates: Iterable[str]) -> Conditions: + """Returns the ways that code compiles where conditions say that the code + that holds it does, and predicates hold too.""" + added = frozenset(predicates) + return _reduced(way | added for way in conditions) + + +@dataclass(frozen=True) +class Entry: + """A test asset that may change: an item, a pseudo-item such as a module's + imports, or a file. Its key identifies it across commits; loose is its + kind and name, and scope the module or file and container that hold it. + content is what counts as a change, and moved what an unchanged copy + elsewhere holds. The content of a monotone entry is a set of lines, which + may grow without changing what it was. conditions are the ways that a + Rust item compiles: for each, the cfg predicates that must hold, its own + and those that its module and the items that hold it inherit, which a move + must keep.""" + + key: tuple[str, ...] + loose: tuple[str, ...] + scope: tuple[str, ...] + name: str + content: str + moved: str + category: str + kind: str + what: str + location: Location + monotone: bool = False + conditions: Conditions = ANYWHERE + + +# How alike the content of a removed entry and of a new one in the same scope +# must be for the new one to count as the old one renamed. +RENAMED_SIMILARITY = 0.6 +_WORDS = re.compile(r"\w+|[^\w\s]") + + +def compare(base: Sequence[Entry], head: Sequence[Entry]) -> list[Change]: + """Compares the entries of two commits: an entry of both is modified when + its content or its conditions changed; one of the base only is relocated + when the head holds its content elsewhere under its conditions, modified + when the head holds its content under others, or an entry of its kind and + name or, in its scope, of content like its own, and removed otherwise; + and one of the head only is added.""" + before = {entry.key: entry for entry in base} + after = {entry.key: entry for entry in head} + changes: list[Change] = [] + for key in sorted(before.keys() & after.keys()): + old, new = before[key], after[key] + same_conditions = old.conditions == new.conditions + if old.content == new.content and same_conditions: + if old.location.path != new.location.path: + changes.append(_change(old, new, "relocated")) + continue + if ( + old.monotone + and same_conditions + and set(old.content.split("\n")) <= set(new.content.split("\n")) + ): + changes.append(_change(old, new, "extended")) + continue + details = [] if old.content == new.content else [_detail(old, new)] + if not same_conditions: + details.append(_conditions_detail(old, new)) + changes.append(_change(old, new, "modified", _join(details))) + removed = [before[key] for key in sorted(before.keys() - after.keys())] + unmatched = {key: after[key] for key in sorted(after.keys() - before.keys())} + by_moved: dict[str, list[Entry]] = defaultdict(list) + by_loose: dict[tuple[str, ...], list[Entry]] = defaultdict(list) + by_scope: dict[tuple[str, ...], list[Entry]] = defaultdict(list) + for entry in unmatched.values(): + by_moved[entry.moved].append(entry) + by_loose[entry.loose].append(entry) + by_scope[entry.scope].append(entry) + + def take(candidates: Sequence[Entry], entry: Entry) -> Entry | None: + """Takes the unmatched candidate most like entry: of its conditions, + then of its kind and name.""" + left = [candidate for candidate in candidates if candidate.key in unmatched] + if not left: + return None + match = min( + left, + key=lambda candidate: ( + candidate.conditions != entry.conditions, + candidate.loose != entry.loose, + ), + ) + del unmatched[match.key] + return match + + left: list[Entry] = [] + for entry in removed: + match = take(by_moved.get(entry.moved, []), entry) + if match is None: + left.append(entry) + elif match.conditions == entry.conditions: + changes.append(_change(entry, match, "relocated")) + else: + detail = _join(["moved", _conditions_detail(entry, match)]) + changes.append(_change(entry, match, "modified", detail)) + for entry in left: + match = take(by_loose.get(entry.loose, []), entry) if entry.name else None + if match is not None: + details = [_detail(entry, match, moved=True)] + if match.conditions != entry.conditions: + details.append(_conditions_detail(entry, match)) + changes.append(_change(entry, match, "modified", _join(details))) + continue + match = _renamed(entry, by_scope.get(entry.scope, []), unmatched) + if match is not None: + del unmatched[match.key] + details = [f"renamed to `{match.name}`"] + if match.conditions != entry.conditions: + details.append(_conditions_detail(entry, match)) + changes.append(_change(entry, match, "modified", _join(details))) + continue + changes.append(_change(entry, None, "removed")) + changes.extend(_change(None, entry, "added") for entry in unmatched.values()) + return changes + + +def _join(details: Sequence[str]) -> str: + return ", and ".join(detail for detail in details if detail) + + +def _conditions_detail(old: Entry, new: Entry) -> str: + """Says how the conditions under which an entry compiles changed.""" + + def listed(conditions: Conditions) -> str: + ways = sorted( + " and ".join(f"`{predicate}`" for predicate in sorted(way)) + for way in conditions + ) + if len(ways) == 1: + return ways[0] + return " or ".join(f"({way})" if " and " in way else way for way in ways) + + if new.conditions == ANYWHERE: + return f"it compiles without {listed(old.conditions)}" + if old.conditions == ANYWHERE: + return f"it now compiles only where {listed(new.conditions)} hold" + return ( + f"it now compiles where {listed(new.conditions)} hold, " + f"not where {listed(old.conditions)} did" + ) + + +def _renamed( + entry: Entry, + candidates: Sequence[Entry], + unmatched: Mapping[tuple[str, ...], Entry], +) -> Entry | None: + """Returns the new entry in entry's scope whose content is most like its + own, if alike enough.""" + if entry.kind not in ("test", "helper"): + return None + words = _WORDS.findall(entry.content) + best: tuple[float, Entry] | None = None + for candidate in candidates: + if candidate.key not in unmatched or candidate.kind != entry.kind: + continue + matcher = difflib.SequenceMatcher( + None, words, _WORDS.findall(candidate.content), autojunk=False + ) + if matcher.quick_ratio() < RENAMED_SIMILARITY: + continue + ratio = matcher.ratio() + if ratio >= RENAMED_SIMILARITY and (best is None or ratio > best[0]): + best = (ratio, candidate) + return None if best is None else best[1] + + +def _change( + old: Entry | None, new: Entry | None, action: str, detail: str = "" +) -> Change: + entry = old if old is not None else new + assert entry is not None + return Change( + entry.category, + action, + entry.kind, + entry.what, + None if old is None else old.location, + None if new is None else new.location, + detail, + ) + + +_PATH_PREFIX = re.compile(r"(?:(? str: + first, second = (old.moved, new.moved) if moved else (old.content, new.content) + paths = old.key[0] == "rust" and _PATH_PREFIX.sub("", first) == _PATH_PREFIX.sub( + "", second + ) + if moved: + return "moved, and only the paths that it names changed" if paths else "moved" + return "only the paths that it names changed" if paths else "" + + +# Rust items. + + +def _tokens(tokens: Sequence[Token], start: int, end: int) -> str: + return _joined(tokens, range(start, end)) + + +# The pairs of punctuation that Rust reads as one operator when nothing +# separates them, which rustfmt keeps together: `a && b` and `a & &b` differ. +_OPERATOR_PAIRS = frozenset( + {"!=", "%=", "&&", "&=", "*=", "+=", "-=", "->", "..", "/=", "::", "<<"} + | {"<=", "==", "=>", ">=", ">>", "^=", "|=", "||"} +) + + +def _joined(tokens: Sequence[Token], indices: Iterable[int]) -> str: + """Joins the texts of the tokens at indices with spaces, but for the + punctuation that forms an operator with the punctuation right before it + in the source, as the second `&` of `&&` does, which joins it.""" + words: list[str] = [] + previous: Token | None = None + for index in indices: + token = tokens[index] + if ( + previous is not None + and previous.kind == PUNCT + and token.kind == PUNCT + and previous.end == token.start + and previous.text + token.text in _OPERATOR_PAIRS + ): + words[-1] += token.text + else: + words.append(token.text) + previous = token + return " ".join(words) + + +# Keywords after which a parenthesized group is an expression or a pattern, +# not the arguments of a call. +_EXPRESSION_KEYWORDS = frozenset( + {"box", "break", "else", "if", "in", "let", "match", "move", "mut", "ref"} + | {"return", "while", "yield"} +) + + +def _one_tuple(parsed: RustFile, comma: int) -> bool: + """Whether the comma before a `)` makes its group a tuple of one element, + whose comma rustfmt never drops.""" + tokens, partner = parsed.tokens, parsed.partner + opener = partner[comma + 1] + if tokens[opener].text != "(": + return False + pieces = layout.split_commas(tokens, partner, opener + 1, comma + 1) + if len(pieces) != 1: + return False + if opener == 0: + return True + before = tokens[opener - 1] + if before.kind == IDENT: + return before.text in _EXPRESSION_KEYWORDS + if ( + before.kind == PUNCT + and before.text == ">" + and layout.is_token(tokens, opener - 2, PUNCT, "-", "=") + and tokens[opener - 2].end == before.start + ): + # After `->` or `=>`, a group is a type or an expression, not the + # arguments of a call: `-> (u8,)` returns a tuple. + return True + return not (before.kind == PUNCT and before.text in "!>)]?") + + +def _statements(parsed: RustFile, opener: int) -> bool: + """Whether the block that opener opens holds a statement: a `;` outside + its inner brackets, or nothing at all.""" + tokens, partner = parsed.tokens, parsed.partner + close = partner[opener] + if close == opener + 1: + return True + index = opener + 1 + while index < close: + token = tokens[index] + if token.kind == PUNCT: + if token.text == ";": + return True + if token.text in "([{": + index = partner[index] + 1 + continue + index += 1 + return False + + +def _match_arms(parsed: RustFile, keyword: int, end: int) -> int | None: + """Returns the brace that opens the arms of the match at keyword.""" + tokens, partner = parsed.tokens, parsed.partner + index = keyword + 1 + while index < end: + token = tokens[index] + if token.kind == PUNCT and token.text in "([": + index = partner[index] + 1 + continue + if token.kind == PUNCT and token.text == "{": + # A block may be the scrutinee, as in `match { x } { ... }`. + after = partner[index] + 1 + if index == keyword + 1 and layout.is_token(tokens, after, PUNCT, "{"): + return after + return index + index += 1 + return None + + +# The macros of the standard library whose arguments are expressions, or a +# pattern, and that accept a trailing comma, in which commas count as they do +# outside macros. +_EXPRESSION_MACROS = frozenset( + {"assert", "assert_eq", "assert_ne", "debug_assert", "debug_assert_eq"} + | {"debug_assert_ne", "dbg", "eprint", "eprintln", "format", "format_args"} + | {"matches", "panic", "print", "println", "todo", "unimplemented"} + | {"unreachable", "vec", "write", "writeln"} +) + + +def _opaque_trees(parsed: RustFile, start: int, end: int) -> list[int]: + """Returns the openers of the token trees from start to end that macros + read as they choose: those of the invocations of the macros but those of + _EXPRESSION_MACROS, and of the macro_rules! definitions.""" + tokens, partner = parsed.tokens, parsed.partner + trees: list[int] = [] + for index in range(max(start, 1), end): + before = tokens[index - 1] + if ( + not layout.is_token(tokens, index, PUNCT, "!") + or before.kind != IDENT + or before.text in _EXPRESSION_KEYWORDS + or before.text in _EXPRESSION_MACROS + ): + continue + opener = index + 1 + if before.text == "macro_rules" and opener < end: + opener += 1 + if ( + opener < end + and layout.is_token(tokens, opener, PUNCT, "(", "[", "{") + and partner[opener] < end + ): + trees.append(opener) + return trees + + +def _formatting(parsed: RustFile, start: int, end: int) -> set[int]: + """Returns the tokens from start to end whose presence rustfmt chooses by + the width that the code has: the commas between the arms of a match, the + braces of a match arm's or closure's body that holds one expression, and + the comma before a closing bracket. In the token tree of + a macro, which rustfmt leaves as it finds it but for its expressions' + match arms and blocks, and whose commas may choose the macro's rule, only + the commas between the arms of a match and the braces of their bodies and + of closures' bodies count as formatting, unless the macro is one of the + standard library's that take expressions and a trailing comma.""" + tokens, partner = parsed.tokens, parsed.partner + opaque: set[int] = set() + for opener in _opaque_trees(parsed, start, end): + opaque.update(range(opener + 1, partner[opener])) + arms_of_matches: set[int] = set() + skipped: set[int] = set() + for index in range(start, end): + token = tokens[index] + if token.kind == IDENT and token.text == "match": + arms = _match_arms(parsed, index, end) + if arms is None: + continue + arms_of_matches.add(arms) + position = arms + 1 + while position < partner[arms]: + if layout.is_token(tokens, position, PUNCT, "(", "[", "{"): + position = partner[position] + 1 + continue + if layout.is_token(tokens, position, PUNCT, ","): + skipped.add(position) + position += 1 + elif token.kind == PUNCT and token.text == "{" and index > 0: + arm = index >= 2 and (tokens[index - 2].text, tokens[index - 1].text) == ( + "=", + ">", + ) + if arm and index in opaque: + # In a macro, `=>` may be the macro's own, as in its rules. + arm = _innermost(parsed, start, index) in arms_of_matches + closure = _closure_body(parsed, index, start) + if (arm or closure) and not _statements(parsed, index): + skipped.update((index, partner[index])) + elif token.kind == PUNCT and token.text == "," and index not in opaque: + closer = layout.is_token(tokens, index + 1, PUNCT, ")", "]", "}") + one_tuple = layout.is_token(tokens, index + 1, PUNCT, ")") and _one_tuple( + parsed, index + ) + if closer and not one_tuple: + skipped.add(index) + return skipped + + +def _closure_body(parsed: RustFile, brace: int, start: int) -> bool: + """Whether the brace at index brace opens the body of a closure, right + after the `|` that ends its parameters, rather than a block that follows + an operator `|`, as in `a | { b }`: the `|` that starts the parameters, + after start, starts an operand.""" + tokens, partner = parsed.tokens, parsed.partner + if not layout.is_token(tokens, brace - 1, PUNCT, "|"): + return False + index = brace - 2 + while index >= start: + token = tokens[index] + if token.kind == PUNCT and token.text in ")]}": + index = partner[index] - 1 + continue + if token.kind == PUNCT and token.text in "([{;": + return False + if token.kind == PUNCT and token.text == "|": + return _operand(tokens, index, start) + index -= 1 + return False + + +def _innermost(parsed: RustFile, start: int, index: int) -> int | None: + """Returns the opener of the innermost bracket group from start that + holds index.""" + tokens, partner = parsed.tokens, parsed.partner + found: int | None = None + position = start + while position < index: + if layout.is_token(tokens, position, PUNCT, "(", "[", "{"): + if partner[position] > index: + found = position + position += 1 + continue + position = partner[position] + 1 + continue + position += 1 + return found + + +def _render(parsed: RustFile, start: int, end: int) -> str: + """Renders tokens as they count for a change: without those whose presence + rustfmt chooses by the width that the code has, so that code that moves to + another indentation renders the same.""" + skipped = _formatting(parsed, start, end) + return _joined( + parsed.tokens, (index for index in range(start, end) if index not in skipped) + ) + + +_INCLUDE = re.compile(r'\b(include_str|include_bytes|include) ! \( "([^"\\]*)" \)') + + +def _resolve_includes(content: str, path: str) -> str: + """Names the file of each include_str!, include_bytes!, or include! from + the repository's root rather than from the file that includes it, so that + the include stays the same when its file moves.""" + + def resolve(match: re.Match[str]) -> str: + relative = match.group(2) + resolved = posixpath.normpath(posixpath.join(posixpath.dirname(path), relative)) + return f'{match.group(1)} ! ( "/{resolved}" )' + + return _INCLUDE.sub(resolve, content) + + +def _after_visibility(parsed: RustFile, index: int) -> int: + if layout.is_token(parsed.tokens, index, IDENT, "pub"): + index += 1 + if layout.is_token(parsed.tokens, index, PUNCT, "("): + index = parsed.partner[index] + 1 + return index + + +def _render_cfg(cfg: layout.Cfg) -> str | None: + if cfg.operator == "option": + return cfg.name if cfg.value is None else f"{cfg.name} = {cfg.value}" + if cfg.operator in ("all", "any", "not"): + operands = [_render_cfg(operand) for operand in cfg.operands] + if any(operand is None for operand in operands): + return None + return f"{cfg.operator} ( {' , '.join(cast(list[str], operands))} )" + return None + + +def _cfg_text(cfg: layout.Cfg) -> str: + """Renders a predicate, with the tokens of what does not parse.""" + if cfg.operator in ("all", "any", "not"): + operands = " , ".join(_cfg_text(operand) for operand in cfg.operands) + return f"{cfg.operator} ( {operands} )" if operands else f"{cfg.operator} ( )" + return _render_cfg(cfg) or " ".join(cfg.words) + + +def _conjuncts( + parsed: RustFile, attributes: Iterable[layout.Attribute] +) -> frozenset[str]: + """Returns the predicates that the cfg attributes among attributes all + require: the operands of an `all`, whose own operands an `all` among them + extends, and any other predicate, each rendered; and for a cfg that + cfg_attr applies where P holds, which nested cfg_attrs extend, `any(not(P), + X)`, where X is what the cfg requires.""" + tokens, partner = parsed.tokens, parsed.partner + found: set[str] = set() + + def add(cfg: layout.Cfg) -> None: + if cfg.operator == "all": + for operand in cfg.operands: + add(operand) + return + found.add(_cfg_text(cfg)) + + def conditional(start: int, end: int, held: tuple[layout.Cfg, ...]) -> None: + """Adds what the cfg attributes that the cfg_attr from start to end + applies require where held holds too.""" + pieces = layout.split_commas(tokens, partner, start, end) + if not pieces: + return + held = (*held, layout.parse_cfg(tokens, partner, *pieces[0])) + where = held[0] if len(held) == 1 else layout.Cfg("all", operands=held) + for begin, finish in pieces[1:]: + if finish - begin < 3 or not layout.is_token(tokens, begin + 1, PUNCT, "("): + continue + inner = (begin + 2, partner[begin + 1]) + if layout.is_token(tokens, begin, IDENT, "cfg"): + required = _cfg_text(layout.parse_cfg(tokens, partner, *inner)) + found.add(f"any ( not ( {_cfg_text(where)} ) , {required} )") + elif layout.is_token(tokens, begin, IDENT, "cfg_attr"): + conditional(*inner, held) + + for attribute in attributes: + if attribute.arguments is None: + continue + if attribute.path == "cfg": + add(layout.parse_cfg(tokens, partner, *attribute.arguments)) + elif attribute.path == "cfg_attr": + conditional(*attribute.arguments, ()) + return frozenset(found) + + +def _assume_test(cfg: layout.Cfg) -> layout.Cfg | None: + """Returns what a predicate requires besides `test`, or None if nothing.""" + if cfg.operator == "option" and cfg.name == "test" and cfg.value is None: + return None + if cfg.operator == "all": + rest = [ + residue + for residue in (_assume_test(operand) for operand in cfg.operands) + if residue is not None + ] + if not rest: + return None + return rest[0] if len(rest) == 1 else layout.Cfg("all", operands=tuple(rest)) + return cfg + + +def _attribute_text(parsed: RustFile, attribute: layout.Attribute, moved: bool) -> str: + """Renders an attribute as it counts for a change: nothing for a lint + attribute, and for moved content, a cfg without its `test`.""" + tokens, partner = parsed.tokens, parsed.partner + paths = layout.attribute_paths(tokens, partner, attribute) + if paths[0] in LINT_ATTRIBUTES or ( + paths[0] == "cfg_attr" and all(path in LINT_ATTRIBUTES for path in paths[1:]) + ): + return "" + if moved and attribute.path == "cfg" and attribute.arguments is not None: + cfg = layout.parse_cfg(tokens, partner, *attribute.arguments) + if layout.requires_test(cfg): + residue = _assume_test(cfg) + if residue is None: + return "" + rendered = _render_cfg(residue) + if rendered is not None: + return f"# [ cfg ( {rendered} ) ]" + return _tokens(tokens, attribute.start, attribute.end + 1) + + +def _item_text( + parsed: RustFile, item: Item, moved: bool, stop: int | None = None +) -> str: + parts = [_attribute_text(parsed, attribute, moved) for attribute in item.attributes] + start = _after_visibility(parsed, item.head) + parts.append(_render(parsed, start, item.end if stop is None else stop)) + return " ".join(part for part in parts if part) + + +def _inner_text(parsed: RustFile, attributes: Iterable[layout.Attribute]) -> str: + return " ".join( + text + for text in ( + _attribute_text(parsed, attribute, False) for attribute in attributes + ) + if text + ) + + +def _use_paths(parsed: RustFile, item: Item) -> list[str]: + """Expands a `use` item into the paths that it imports.""" + tokens, partner = parsed.tokens, parsed.partner + start = _after_visibility(parsed, item.head) + 1 + end = ( + item.end - 1 if layout.is_token(tokens, item.end - 1, PUNCT, ";") else item.end + ) + + def tree(first: int, last: int, prefix: Sequence[str]) -> list[str]: + parts = list(prefix) + index = first + while index < last: + token = tokens[index] + if token.kind == PUNCT and token.text == "{": + found: list[str] = [] + for begin, finish in layout.split_commas( + tokens, partner, index + 1, partner[index] + ): + found.extend(tree(begin, finish, parts)) + return found + if token.kind == PUNCT and token.text == "*": + return ["::".join([*parts, "*"])] + if token.kind == IDENT and token.text == "as" and index + 1 < last: + return ["::".join(parts) + " as " + tokens[index + 1].text] + if token.kind == IDENT: + parts.append(layout.unraw(token.text)) + index += 1 + if parts and parts[-1] == "self": + parts.pop() + return ["::".join(parts)] + + return tree(start, end, ()) + + +_BLOCK_STARTS = frozenset( + {"async", "const", "enum", "fn", "for", "if", "impl", "loop", "match", "mod"} + | {"static", "struct", "trait", "type", "unsafe", "use", "while"} +) + + +def _context(parsed: RustFile, item: Item, limit: int) -> str: + """Returns what the bracket group of item that closes at limit holds: + "statements", those of a block; "types", the fields or variants of a type + or the parameters of a function; or "expressions", such as match arms or + the fields of a struct expression.""" + tokens, partner = parsed.tokens, parsed.partner + if limit >= item.end or not layout.is_token(tokens, limit, PUNCT, ")", "]", "}"): + return "expressions" + opener = partner[limit] + if tokens[opener].text == "{" and _statements(parsed, opener): + return "statements" + if item.kind in ("struct", "union", "enum"): + return "types" + if item.kind == "fn" and (item.body is None or opener < item.body[0]): + return "types" + return "expressions" + + +# The keywords after which an operand starts. +_OPERAND_KEYWORDS = frozenset({"break", "in", "move", "return", "yield"}) + + +def _operand(tokens: Sequence[Token], index: int, first: int) -> bool: + """Whether the token at index starts an operand, as the `|` of a closure + or the `<` of a qualified path do, rather than following one, as a binary + operator does.""" + if index == first: + return True + before = tokens[index - 1] + if before.kind == IDENT: + return before.text in _OPERAND_KEYWORDS + if before.kind != PUNCT: + return False + if before.text == ">": + return layout.is_token(tokens, index - 2, PUNCT, "=") + return before.text in "([{,=:;" + + +# The constructs whose condition, scrutinee, or iterator precedes their body, +# and the keywords after which a `{` opens a block operand, not a body. +_CONDITIONS = frozenset({"for", "if", "match", "while"}) +_CONDITION_KEYWORDS = frozenset( + {"break", "if", "in", "let", "match", "return", "while", "yield"} +) + + +def _ends_operand(tokens: Sequence[Token], index: int) -> bool: + """Whether the token before index ends an operand, as an identifier, a + literal, or a closing bracket does, so that the `{` at index ends a + condition and opens the body after it.""" + if index == 0: + return False + before = tokens[index - 1] + if before.kind == LITERAL: + return True + if before.kind == IDENT: + return before.text not in _CONDITION_KEYWORDS + return before.kind == PUNCT and before.text in ")]}?" + + +def _lone_equals(tokens: Sequence[Token], index: int) -> bool: + """Whether the `=` at index stands alone, as that of a `let` does, rather + than in an operator such as `==`, `<=`, `=>`, or `..=`.""" + equals = tokens[index] + before = tokens[index - 1] if index > 0 else None + after = tokens[index + 1] if index + 1 < len(tokens) else None + joined_before = ( + before is not None + and before.kind == PUNCT + and before.end == equals.start + and before.text in "=!<>.+-*/%^&|" + ) + joined_after = ( + after is not None + and after.kind == PUNCT + and equals.end == after.start + and after.text in "=>" + ) + return not (joined_before or joined_after) + + +def _construct_end( + parsed: RustFile, start: int, limit: int, context: str = "expressions" +) -> int: + """Returns the index after the field, variant, statement, match arm, or + expression that starts at start, after its attributes, within limit, in a + group of the context that _context names. A `;` ends it, and so does a + `,`, but for one in a block of statements, which only generic arguments + and closures' parameters hold, and one in the generic arguments of a type, + a turbofish, or a qualified path, or in a closure's parameters.""" + tokens, partner = parsed.tokens, parsed.partner + index = start + while index < limit: + attribute = layout.attribute_at(tokens, partner, index) + if attribute is None: + break + index = attribute.end + 1 + first = index + first_text = tokens[first].text if first < limit else "" + conditional = first_text in _CONDITIONS + + def scan(generics: bool) -> tuple[int, bool]: + """Returns the end, and whether a `<` that it took to open generic + arguments stayed open.""" + index = first + angles = 0 + # Whether a pattern runs at index, from `let` to its `=` or from `for` + # to `in`, whose braces open no body. + pattern = first_text == "for" + while index < limit: + token = tokens[index] + if token.kind == IDENT and conditional: + if token.text == "let": + pattern = True + elif token.text == "in" and first_text == "for": + pattern = False + if token.kind != PUNCT: + index += 1 + continue + text = token.text + if pattern and text == "=" and _lone_equals(tokens, index): + pattern = False + if text == ";" or (text == "," and not angles and context != "statements"): + return index + 1, False + if text in ")]}": + return index, angles > 0 + if text in "([": + index = partner[index] + 1 + continue + if text == "{": + close = partner[index] + after = close + 1 + if angles or ( + conditional and (pattern or not _ends_operand(tokens, index)) + ): + # The braces of a const argument, as in `S<{ 1 }>`, of a + # pattern, or of a block in a condition, as in + # `if let Some(value) = { take() } { ... }`. + index = after + continue + if layout.is_token(tokens, after, IDENT, "else"): + index = after + 1 + continue + arm = ( + index >= 2 + and tokens[index - 1].text == ">" + and tokens[index - 2].text == "=" + ) + invocation = index >= 1 and tokens[index - 1].text == "!" + if ( + arm + or invocation + or first_text in _BLOCK_STARTS + or first_text == "{" + ): + if layout.is_token(tokens, after, PUNCT, ";", ","): + return after + 1, False + return after, False + index = after + continue + if ( + generics + and text == "<" + and (context == "types" or angles or _operand(tokens, index, first)) + ): + angles += 1 + elif ( + generics + and text == ">" + and angles + and not layout.is_token(tokens, index - 1, PUNCT, "-", "=") + ): + angles -= 1 + elif ( + text == "|" + and context == "expressions" + and not angles + and _operand(tokens, index, first) + ): + # A closure's parameters, which end with the next `|`. + bar = index + 1 + while bar < limit and not layout.is_token(tokens, bar, PUNCT, "|"): + opens = layout.is_token(tokens, bar, PUNCT, "(", "[", "{") + bar = partner[bar] + 1 if opens else bar + 1 + if bar < limit: + index = bar + 1 + continue + index += 1 + return limit, angles > 0 + + end, open_angles = scan(context != "statements") + if open_angles: + # A `<` that no `>` closes is no generic argument list. + end = scan(False)[0] + return end + + +class _RustEntries: + """Collects the test entries of a Rust file, whose module compiles in the + ways that inherited says.""" + + def __init__( + self, + parsed: RustFile, + crate: str, + path: str, + category: str, + inherited: Conditions = ANYWHERE, + ) -> None: + self.parsed = parsed + self.crate = crate + self.path = path + self.category = category + self.ways = inherited + # The predicates that the file's inner attributes add to every way. + self.inherited = _conjuncts(parsed, parsed.inner_attributes) + self.entries: list[Entry] = [] + # The imports of each module, as they count for a change and as they + # count for a move, with the line of the first and the module's + # conditions. + self.imports: dict[ + tuple[str, ...], tuple[set[str], set[str], int, frozenset[str]] + ] = {} + self.seen: dict[tuple[str, ...], int] = defaultdict(int) + + def add( + self, + module: tuple[str, ...], + container: str, + kind: str, + name: str, + cfg: str, + content: str, + moved: str, + test: bool, + line: int, + what: str, + conditions: frozenset[str], + ) -> None: + base = ("rust", self.crate, "::".join(module), container, kind, name, cfg) + self.seen[base] += 1 + key = (*base, str(self.seen[base])) + self.entries.append( + Entry( + key, + ("rust", kind, name), + base[:5], + name, + content, + moved, + self.category, + "test" if test else _KINDS.get(kind, "helper"), + what, + Location(self.path, line), + kind == "use", + _within(self.ways, conditions), + ) + ) + + def item( + self, + module: tuple[str, ...], + container: str, + item: Item, + test: bool, + conditions: frozenset[str], + stop: int | None = None, + extra: str = "", + ) -> None: + parsed = self.parsed + cfg = " & ".join( + sorted( + _tokens(parsed.tokens, attribute.start, attribute.end + 1) + for attribute in item.attributes + if attribute.path == "cfg" + ) + ) + content = _resolve_includes(_item_text(parsed, item, False, stop), self.path) + moved = _resolve_includes(_item_text(parsed, item, True, stop), self.path) + content += extra + moved += extra + name = item.name or _tokens( + parsed.tokens, item.head, min(item.head + 6, item.end) + ) + self.add( + module, + container, + item.kind, + name, + cfg, + content, + moved, + test, + item.line, + _what(module, container, item.kind, name), + conditions | _conjuncts(parsed, item.attributes), + ) + + def inner( + self, + module: tuple[str, ...], + attributes: Sequence[layout.Attribute], + line: int, + conditions: frozenset[str], + ) -> None: + """Adds the inner attributes of a module of test code, even none, so + that one which a change adds, such as a cfg, modifies them.""" + text = _inner_text(self.parsed, attributes) + self.add( + module, + "", + "inner", + "", + "", + text, + text, + False, + line, + f"the inner attributes of `{_module_name(module)}`", + conditions, + ) + + def use( + self, module: tuple[str, ...], item: Item, conditions: frozenset[str] + ) -> None: + """Adds the paths that a `use` item imports to its module's imports, + each with the attributes of the item but its lint attributes, and, as + they count for a move, without `test` in its cfg.""" + parsed = self.parsed + content, moved, line, _ = self.imports.get( + module, (set[str](), set[str](), item.line, conditions) + ) + attributes, moved_attributes = ( + " ".join( + text + for text in ( + _attribute_text(parsed, attribute, for_move) + for attribute in item.attributes + ) + if text + ) + for for_move in (False, True) + ) + for path in _use_paths(parsed, item): + content.add(f"{attributes} {path}".strip()) + moved.add(f"{moved_attributes} {path}".strip()) + self.imports[module] = (content, moved, line, conditions) + + def finish(self) -> list[Entry]: + for module, (content, moved, line, conditions) in sorted(self.imports.items()): + self.add( + module, + "", + "use", + "", + "", + "\n".join(sorted(content)), + "\n".join(sorted(moved)), + False, + line, + f"the imports of `{_module_name(module)}`", + conditions, + ) + return self.entries + + def walk( + self, + items: Sequence[Item], + module: tuple[str, ...], + container: str, + in_test: bool, + conditions: frozenset[str], + ) -> None: + parsed = self.parsed + tokens, partner = parsed.tokens, parsed.partner + for item in items: + # A test function compiles only for tests, whatever guards it. + gated = ( + in_test + or any( + layout.attribute_requires_test(tokens, partner, attribute) + for attribute in item.attributes + ) + or _test_function(parsed, item) + ) + held = conditions | _conjuncts(parsed, item.attributes) + if item.kind == "mod": + inner = (*module, item.name) + if item.body is not None: + # A module's inner attributes guard it as its outer ones do. + gated = gated or any( + layout.attribute_requires_test(tokens, partner, attribute) + for attribute in item.inner_attributes + ) + held |= _conjuncts(parsed, item.inner_attributes) + if gated: + keyword = _after_visibility(parsed, item.head) + self.item(module, container, item, False, conditions, keyword + 2) + if item.body is not None: + self.inner(inner, item.inner_attributes, item.line, held) + self.walk(item.children, inner, "", True, held) + elif item.body is not None: + self.walk(item.children, inner, "", False, held) + continue + if not gated: + self.hooks(module, container, item, conditions) + if item.children: + self.walk(item.children, module, _container(item), False, held) + continue + if item.kind == "use": + self.use(module, item, conditions) + continue + if item.body is not None and ( + item.kind in ("impl", "trait", "extern") + or (item.kind == "macro" and item.children) + ): + inner_text = _inner_text(parsed, item.inner_attributes) + self.item( + module, + container, + item, + False, + conditions, + item.body[0], + f" {inner_text}" if inner_text else "", + ) + self.walk(item.children, module, _container(item), True, held) + continue + test = _test_function(parsed, item) or ( + item.kind == "fn" and container == "proptest!" + ) + self.item(module, container, item, test, conditions) + + def hooks( + self, + module: tuple[str, ...], + container: str, + item: Item, + conditions: frozenset[str], + ) -> None: + """Adds the test hooks of a production item, but those of the items + that it holds.""" + parsed = self.parsed + tokens, partner = parsed.tokens, parsed.partner + held = [(child.start, child.end) for child in item.children] + + def outside(index: int) -> bool: + return item.start <= index < item.end and not any( + start <= index < end for start, end in held + ) + + pieces: list[tuple[int, str]] = [] + for attribute in parsed.attributes: + if not outside(attribute.start) or not layout.attribute_mentions_test( + tokens, partner, attribute + ): + continue + end = attribute.end + 1 + if attribute.start >= item.head and layout.attribute_requires_test( + tokens, partner, attribute + ): + limit = _limit(parsed, item, attribute.start) + end = _construct_end( + parsed, attribute.start, limit, _context(parsed, item, limit) + ) + pieces.append((attribute.start, _render(parsed, attribute.start, end))) + for index, cfg in parsed.cfg_macros: + if outside(index) and layout.mentions_test(cfg): + pieces.append((index, _tokens(tokens, index, partner[index + 2] + 1))) + if not pieces: + return + text = _resolve_includes( + " | ".join(piece for _, piece in sorted(pieces)), self.path + ) + name = f"{item.kind} {item.name}".strip() + self.add( + module, + container, + "hooks", + name, + "", + text, + text, + False, + item.line, + f"the test hooks of `{name}` in `{_module_name(module)}`" + + (f" (`{container}`)" if container else ""), + conditions | _conjuncts(parsed, item.attributes), + ) + + +_KINDS = {"hooks": "hook", "inner": "attributes", "mod": "module", "use": "imports"} + + +def _test_function(parsed: RustFile, item: Item) -> bool: + """Whether an attribute of item, or one that a cfg_attr of it applies, + makes it a test, as #[test] does.""" + return any( + layout.is_test_function_attribute(path) + for attribute in item.attributes + for path in layout.attribute_paths(parsed.tokens, parsed.partner, attribute) + ) + + +def _limit(parsed: RustFile, item: Item, index: int) -> int: + """Returns the end of the innermost bracket group of item that holds + index.""" + tokens, partner = parsed.tokens, parsed.partner + limit = item.end + position = item.start + while position < limit: + if ( + tokens[position].kind == PUNCT + and tokens[position].text in "([{" + and position < index < partner[position] + ): + limit = partner[position] + position += 1 + continue + if tokens[position].kind == PUNCT and tokens[position].text in "([{": + position = partner[position] + 1 + continue + position += 1 + return limit + + +def _container(item: Item) -> str: + if item.kind == "impl": + return item.name + if item.kind == "trait": + return f"trait {item.name}" + if item.kind == "extern": + return "extern" + return f"{item.name}!" + + +def _module_name(module: tuple[str, ...]) -> str: + if module and module[0] == "[tests]": + return "/".join(module[1:]) + " (tests)" + return "::".join(module) or "crate" + + +def _what(module: tuple[str, ...], container: str, kind: str, name: str) -> str: + where = _module_name(module) + (f", `{container}`" if container else "") + # An impl's name is its header, which starts with its kind. + item = name if kind == "impl" else f"{kind} {name}" + return f"`{item}` in `{where}`" + + +def rust_module(path: str, crate: str) -> tuple[str, ...]: + """Returns the module path of a Rust file of a crate, from its root; a file + of the crate's tests directory has one of its own.""" + relative = layout.crate_relative(path, crate) + if relative.startswith("src/"): + return layout.module_path(relative[len("src/") :]) + return ("[tests]", *relative[len("tests/") : -len(".rs")].split("/")) + + +def rust_entries( + path: str, + crate: str, + text: str, + category: str, + whole: bool, + inherited: Conditions = ANYWHERE, +) -> list[Entry]: + """Returns the test entries of a Rust file, whose module compiles in the + ways that inherited says: all of its items when whole, as for a test file + or a file whose inner attributes require `test`, and otherwise its test + code and test hooks. Raises RustSyntaxError when the file does not parse.""" + parsed = layout.parse_rust(text) + module = rust_module(path, crate) + collector = _RustEntries(parsed, crate, path, category, inherited) + whole = whole or any( + layout.attribute_requires_test(parsed.tokens, parsed.partner, attribute) + for attribute in parsed.inner_attributes + ) + if whole: + collector.inner(module, parsed.inner_attributes, 1, collector.inherited) + else: + pieces = [ + _tokens(parsed.tokens, attribute.start, attribute.end + 1) + for attribute in parsed.inner_attributes + if layout.attribute_mentions_test(parsed.tokens, parsed.partner, attribute) + ] + if pieces: + text = " | ".join(pieces) + collector.add( + module, + "", + "hooks", + "", + "", + text, + text, + False, + 1, + f"the test hooks of `{_module_name(module)}`", + collector.inherited, + ) + collector.walk(parsed.items, module, "", whole, collector.inherited) + return collector.finish() + + +# Python items. + + +def _without_docstring(body: list[ast.stmt]) -> list[ast.stmt]: + if ( + body + and isinstance(body[0], ast.Expr) + and isinstance(body[0].value, ast.Constant) + and isinstance(body[0].value.value, str) + ): + return body[1:] + return body + + +def python_entries(path: str, text: str, category: str) -> list[Entry]: + """Returns the entries of a Python module: its functions, classes and + their methods, assignments, imports, and other statements. Raises + SyntaxError or ValueError when it does not parse.""" + tree = ast.parse(text, path) + entries: list[Entry] = [] + seen: dict[tuple[str, ...], int] = defaultdict(int) + imports: set[str] = set() + others: list[str] = [] + first_other = 0 + + def add( + container: str, kind: str, name: str, content: str, line: int, label: str + ) -> None: + base = ("python", path, container, kind, name) + seen[base] += 1 + where = f"{container}." if container else "" + what = ( + f"`{kind} {where}{name}` in `{path}`" if name else f"the {kind} of `{path}`" + ) + entries.append( + Entry( + (*base, str(seen[base])), + ("python", kind, name), + base[:4], + name, + content, + content, + category, + label, + what, + Location(path, line), + kind == "imports", + ) + ) + + def visit(node: ast.stmt, container: str) -> None: + nonlocal first_other + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + function = copy.copy(node) + function.body = _without_docstring(node.body) + test = ( + node.name.startswith("test") + if container + else node.name.startswith("test_") + ) + add( + container, + "def", + node.name, + ast.dump(function), + node.lineno, + "test" if test else "helper", + ) + elif isinstance(node, ast.ClassDef): + header = copy.copy(node) + header.body = [ + statement + for statement in _without_docstring(node.body) + if not isinstance( + statement, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef) + ) + ] + add(container, "class", node.name, ast.dump(header), node.lineno, "helper") + inner = f"{container}.{node.name}" if container else node.name + for statement in node.body: + if isinstance( + statement, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef) + ): + visit(statement, inner) + elif isinstance(node, ast.Import): + imports.update( + f"import {alias.name} as {alias.asname}" for alias in node.names + ) + elif isinstance(node, ast.ImportFrom): + source = "." * node.level + (node.module or "") + imports.update( + f"from {source} import {alias.name} as {alias.asname}" + for alias in node.names + ) + elif isinstance(node, (ast.Assign, ast.AnnAssign)) and not container: + targets = node.targets if isinstance(node, ast.Assign) else [node.target] + name = ", ".join(ast.unparse(target) for target in targets) + add("", "assignment", name, ast.dump(node), node.lineno, "helper") + else: + if not others: + first_other = node.lineno + others.append(ast.dump(node)) + + for statement in _without_docstring(tree.body): + visit(statement, "") + if imports: + add("", "imports", "", "\n".join(sorted(imports)), 1, "imports") + if others: + add("", "statements", "", "\n".join(others), first_other, "statements") + return entries + + +def file_entry(path: str, blob: str, category: str) -> Entry: + """Returns a file as an entry, whose content is its blob.""" + directory, _, name = path.rpartition("/") + return Entry( + ("file", path), + ("file", name), + ("file", directory), + name, + blob, + blob, + category, + "file", + f"`{path}`", + Location(path, 0), + ) + + +# Test policy and CI wiring. + + +def allowlist_changes(base: str | None, head: str | None) -> list[Change]: + """Returns the allowlist entries that are new or allow more. The first + allowlist loosens nothing that a test policy held before.""" + if base is None or head is None: + return [] + before = layout.parse_allowlist(base)[0] + after, errors = layout.parse_allowlist(head) + location = Location(layout.ALLOWLIST, 0) + if errors: + return [ + Change( + TEST_POLICY, + "modified", + "file", + f"`{layout.ALLOWLIST}`", + location, + location, + "it does not parse: " + "; ".join(errors), + ) + ] + + changes: list[Change] = [] + + def loosened(path: str, detail: str) -> None: + changes.append( + Change( + TEST_POLICY, + "loosened", + "allowlist entry", + f"`{path}`", + None, + location, + detail, + ) + ) + + # The baseline allows test code besides hooks, and its two numbers bound + # different things, so each counts on its own. + for path, (hooks, tests) in sorted(after.baseline.items()): + if path in before.baseline: + old_hooks, old_tests = before.baseline[path] + if hooks > old_hooks or tests > old_tests: + loosened( + path, + f"the baseline allows {hooks} test hooks and {tests} tests, " + f"up from {old_hooks} and {old_tests}", + ) + elif path in before.seams: + loosened(path, "moves from the seams to the baseline, which allows tests") + else: + loosened(path, f"the baseline allows {hooks} test hooks and {tests} tests") + for path, count in sorted(after.seams.items()): + if path in before.seams: + old = before.seams[path] + elif path in before.baseline: + old = before.baseline[path][0] + else: + loosened(path, f"the seams allow {count} test hooks") + continue + if count > old: + loosened(path, f"the seams allow {count} test hooks, up from {old}") + return changes + + +def _without_comment(line: str) -> str: + """Returns a line of YAML, or of the shell or PowerShell script of a step, + without its comment: what follows a `#` that starts the line or a word + outside quotes.""" + quote = "" + for index, character in enumerate(line): + if quote: + if character == quote: + quote = "" + continue + if character in "'\"": + quote = character + elif character == "#" and (index == 0 or line[index - 1] in " \t"): + return line[:index] + return line + + +def _normalized(line: str) -> str: + """Returns a line with its indentation, which nests a YAML key, and its + other runs of whitespace as single spaces.""" + text = _without_comment(line).rstrip() + words = " ".join(text.split()) + return " " * (len(text) - len(text.lstrip(" "))) + words if words else "" + + +def _normalized_lines(text: str) -> list[tuple[int, str]]: + lines: list[tuple[int, str]] = [] + for number, line in enumerate(text.split("\n"), 1): + normalized = _normalized(line) + if normalized: + lines.append((number, normalized)) + return lines + + +def _extends(old: str, new: str) -> bool: + """Whether a line only gains test modules, as a test command that runs + another one does, at the same indentation.""" + if len(old) - len(old.lstrip(" ")) != len(new) - len(new.lstrip(" ")): + return False + before, after = Counter(old.split()), Counter(new.split()) + if before - after: + return False + return all(TEST_MODULE.fullmatch(word) for word in (after - before).elements()) + + +def line_changes(path: str, base: str | None, head: str | None) -> list[Change]: + """Returns the removed and changed lines of a test action or workflow, but + those that only gain test modules, and its added lines that may keep tests + from running.""" + if base is None: + return [] + if head is None: + return [Change(CI_WIRING, "removed", "file", f"`{path}`", Location(path, 0))] + return _line_diff(path, _normalized_lines(base), _normalized_lines(head)) + + +def _line_diff( + path: str, old: Sequence[tuple[int, str]], new: Sequence[tuple[int, str]] +) -> list[Change]: + """Returns the removed and changed lines of old in new, but those that + only gain test modules, and the added lines that may keep tests from + running.""" + matcher = difflib.SequenceMatcher( + None, [line for _, line in old], [line for _, line in new], autojunk=False + ) + changes: list[Change] = [] + for tag, first, last, begin, end in matcher.get_opcodes(): + if ( + tag == "replace" + and last - first == end - begin + and all( + _extends(before[1], after[1]) + for before, after in zip(old[first:last], new[begin:end], strict=True) + ) + ): + continue + if tag in ("replace", "delete"): + for number, line in old[first:last]: + changes.append( + Change( + CI_WIRING, + "modified" if tag == "replace" else "removed", + "line", + f"`{_shorten(line.strip())}`", + Location(path, number), + Location(path, new[begin][0]) if begin < end else None, + ) + ) + if tag in ("replace", "insert"): + for number, line in new[begin:end]: + if SKIPPING.search(line): + changes.append( + Change( + CI_WIRING, + "added a line that may skip", + "line", + f"`{_shorten(line.strip())}`", + None, + Location(path, number), + ) + ) + return changes + + +def _shorten(text: str, limit: int = 120) -> str: + text = text.replace("`", "'") + return text if len(text) <= limit else text[: limit - 3] + "..." + + +def wiring_tokens(text: str) -> dict[str, list[int]]: + """Returns the test actions, workflows, commands, modules, and scenarios + that a workflow names, with the line of each time that it names one, as + ci.yml names validate-nvx once for each operating system. A line that ends + with a backslash continues on the next.""" + logical: list[tuple[int, str]] = [] + pending: list[str] = [] + start = 1 + for number, line in enumerate(text.split("\n"), 1): + line = _without_comment(line) + if not pending: + start = number + if line.rstrip().endswith("\\"): + pending.append(line.rstrip()[:-1]) + continue + logical.append((start, " ".join([*pending, line]))) + pending = [] + if pending: + logical.append((start, " ".join(pending))) + tokens: dict[str, list[int]] = defaultdict(list) + for number, line in logical: + for kind, pattern in enumerate(WIRING_TOKENS): + for match in pattern.finditer(line): + value = re.sub(r'"\+\$\{\{[^}]*\}\}"', "", match.group(1)) + value = " ".join(value.replace("\\", "/").split()) + tokens[f"{kind}:{value}"].append(number) + return dict(tokens) + + +def token_changes(path: str, base: str | None, head: str | None) -> list[Change]: + """Returns the test actions, workflows, commands, modules, and scenarios + that a workflow names fewer times, once for each time that it names one + no more.""" + if base is None: + return [] + old = wiring_tokens(base) + new = wiring_tokens(head or "") + changes: list[Change] = [] + for token, lines in sorted(old.items()): + kept = len(new.get(token, [])) + changes.extend( + Change( + CI_WIRING, + "removed", + "test wiring", + f"`{_shorten(token.split(':', 1)[1])}`", + Location(path, line), + detail=f"named {len(lines)} times before, {kept} now" if kept else "", + ) + for line in lines[kept:] + ) + return changes + + +@dataclass +class WorkflowJob: + """A job of a workflow: the line of its ID, the lines of its keys but its + steps, and the lines of each step, without comments, as _normalized_lines + returns them.""" + + line: int + keys: list[tuple[int, str]] = field(default_factory=list[tuple[int, str]]) + steps: list[list[tuple[int, str]]] = field( + default_factory=list[list[tuple[int, str]]] + ) + + def runs_tests(self) -> bool: + return any(map(_names_tests, [self.keys, *self.steps])) + + def test_lines(self) -> list[tuple[int, str]]: + """Returns the lines of its keys and of its steps that run tests.""" + lines = list(self.keys) + for step in self.steps: + if _names_tests(step): + lines.extend(step) + return lines + + +def _names_tests(lines: Sequence[tuple[int, str]]) -> bool: + return bool(wiring_tokens("\n".join(text for _, text in lines))) + + +def workflow_jobs(text: str) -> dict[str, WorkflowJob]: + """Returns the jobs of a workflow by their IDs.""" + jobs: dict[str, WorkflowJob] = {} + job: WorkflowJob | None = None + in_jobs = False + job_indent: int | None = None + key_indent: int | None = None + step_indent: int | None = None + in_steps = False + for number, raw in enumerate(text.split("\n"), 1): + line = _without_comment(raw).rstrip() + stripped = line.strip() + if not stripped: + continue + depth = len(line) - len(line.lstrip(" ")) + if depth == 0: + in_jobs = line == "jobs:" + job = None + continue + if not in_jobs: + continue + if job_indent is None: + job_indent = depth + if depth <= job_indent: + job = jobs[stripped.removesuffix(":").strip("'\"")] = WorkflowJob(number) + key_indent = step_indent = None + in_steps = False + continue + if job is None: + continue + if key_indent is None: + key_indent = depth + normalized = (number, _normalized(raw)) + if in_steps: + item = stripped.startswith("- ") or stripped == "-" + if item and depth >= key_indent and depth == (step_indent or depth): + step_indent = depth + job.steps.append([normalized]) + continue + if depth > (key_indent if step_indent is None else step_indent): + if job.steps: + job.steps[-1].append(normalized) + continue + in_steps = False + if depth == key_indent and stripped == "steps:": + in_steps = True + step_indent = None + continue + job.keys.append(normalized) + return jobs + + +def job_changes(path: str, base: str | None, head: str | None) -> list[Change]: + """Returns, for the jobs of a workflow that run tests, the removed and + changed lines of their keys and of their steps that run tests, but those + that only gain test modules, and the lines added there that may keep tests + from running, such as an `if:`, a `continue-on-error`, or a conditional + that wraps a command; and the jobs that run tests that the workflow no + longer has.""" + if base is None: + return [] + old, new = workflow_jobs(base), workflow_jobs(head or "") + changes: list[Change] = [] + for name, job in sorted(old.items()): + if not job.runs_tests(): + continue + if name not in new: + changes.append( + Change( + CI_WIRING, + "removed", + "test job", + f"`{name}`", + Location(path, job.line), + ) + ) + continue + changes.extend(_line_diff(path, job.test_lines(), new[name].test_lines())) + return changes + + +def shard_scenarios(text: str) -> set[tuple[str, str]] | None: + """Returns the backends and scenarios of the microVM test shards.""" + try: + data: object = json.loads(text) + except json.JSONDecodeError: + return None + scenarios: set[tuple[str, str]] = set() + if not isinstance(data, dict): + return None + backends = cast(dict[str, object], data).get("backends") + if not isinstance(backends, dict): + return None + for backend, plan in cast(dict[str, object], backends).items(): + shards = ( + cast(dict[str, object], plan).get("shards") + if isinstance(plan, dict) + else None + ) + if not isinstance(shards, list): + return None + for shard in cast(list[object], shards): + listed = ( + cast(dict[str, object], shard).get("scenarios") + if isinstance(shard, dict) + else None + ) + if not isinstance(listed, list): + return None + scenarios.update( + (backend, str(scenario)) for scenario in cast(list[object], listed) + ) + return scenarios + + +def shard_changes(base: str | None, head: str | None) -> list[Change]: + """Returns the scenarios that leave a backend's shards.""" + if base is None: + return [] + old = shard_scenarios(base) + new = shard_scenarios(head or "{}") if head is not None else set[tuple[str, str]]() + location = Location(SHARDS_FILE, 0) + if old is None or new is None: + return [ + Change( + CI_WIRING, + "modified", + "file", + f"`{SHARDS_FILE}`", + location, + location, + "it does not parse", + ) + ] + return [ + Change( + CI_WIRING, + "removed", + "scenario", + f"`{scenario}` on {backend}", + location, + ) + for backend, scenario in sorted(old - new) + ] + + +# Commits. + + +class Tree: + """The blobs of a commit, by path.""" + + def __init__(self, root: Path, commit: str) -> None: + self.commit = commit + self.blobs: dict[str, str] = {} + listing = layout.git(root, "ls-tree", "-r", "-z", "--full-tree", commit) + for entry in listing.split(b"\0"): + if not entry: + continue + metadata, _, name = entry.partition(b"\t") + fields = metadata.decode("ascii").split() + if fields[1] == "blob": + self.blobs[os.fsdecode(name)] = fields[2] + + +def read_blobs(root: Path, blobs: Iterable[str]) -> dict[str, bytes]: + """Reads blobs by their object names, with one git process.""" + names = sorted(set(blobs)) + if not names: + return {} + output = layout.git( + root, "cat-file", "--batch", stdin=("\n".join(names) + "\n").encode("ascii") + ) + found: dict[str, bytes] = {} + position = 0 + for name in names: + end = output.index(b"\n", position) + header = output[position:end].decode("ascii").split() + if len(header) != 3: + raise FlagError(f"git cat-file cannot read {name}") + size = int(header[2]) + found[name] = output[end + 1 : end + 1 + size] + position = end + 1 + size + 1 + return found + + +def resolve(root: Path, revision: str) -> str: + """Returns the commit that a revision names.""" + output = layout.git(root, "rev-parse", "--verify", f"{revision}^{{commit}}") + return output.decode("ascii").strip() + + +@dataclass +class Comparison: + """The changes to the test assets between a merge base and a head.""" + + base: str + head: str + changes: list[Change] = field(default_factory=list[Change]) + + @property + def flagged(self) -> list[Change]: + return sorted( + (change for change in self.changes if change.flags), key=Change.order + ) + + def count(self, action: str) -> int: + return sum(1 for change in self.changes if change.action == action) + + +def classify(path: str, crates: Sequence[str]) -> tuple[str, str] | None: + """Returns the category of a test asset and how to compare it: by its + "rust" items, its "rust-production" test code, its "python" items, its + "file", its "lines", its "tokens", its "shards", or its "allowlist" + entries; or None for a file that is no test asset.""" + if path == layout.ALLOWLIST: + return TEST_POLICY, "allowlist" + if path in POLICY_FILES: + return TEST_POLICY, "file" + if path in WIRING_FILES: + return CI_WIRING, "lines" + if path in WIRING_TOKEN_FILES: + return CI_WIRING, "tokens" + if path == SHARDS_FILE: + return CI_WIRING, "shards" + if DOUBLES.fullmatch(path): + return TEST_DOUBLES, "rust" + if END_TO_END_FILES.fullmatch(path): + return END_TO_END, "file" + if layout.PYTHON_END_TO_END_TESTS.fullmatch(path): + return END_TO_END, "python" + if layout.PYTHON_UNIT_TESTS.fullmatch(path) or path in layout.PYTHON_TEST_SUPPORT: + return TEST_CODE, "python" + if TEST_DATA_FILES.fullmatch(path): + return TEST_DATA, "file" + crate = layout.crate_of(path, crates) + if crate is None: + return None + relative = layout.crate_relative(path, crate) + if path.endswith(".md"): + return None + if relative.startswith(layout.TEST_DATA_DIRECTORIES): + return TEST_DATA, "file" + role = layout.rust_role(path, crate) + if role == "test": + return TEST_CODE, "rust" + if role == "production": + return TEST_CODE, "rust-production" + if relative.startswith("tests/") or "/tests/" in relative: + return TEST_DATA, "file" + return None + + +def _crate_path(crate: str, relative: str) -> str: + return f"{crate}/{relative}" if crate else relative + + +def _target_root(relative: str) -> bool: + """Whether a file, relative to its crate, roots one of the crate's + targets: the library, a binary, a test, a benchmark, an example, or the + build script.""" + directory, name = posixpath.split(relative) + if relative in ("src/lib.rs", "src/main.rs", "build.rs"): + return True + return any( + directory == targets + or (posixpath.dirname(directory) == targets and name == "main.rs") + for targets in ("src/bin", "tests", "benches", "examples") + ) + + +def _module_directory(relative: str, root: bool | None = None) -> str: + """Returns the directory of the modules that a file, relative to its + crate, declares: its own for a target's root or a mod.rs file, and the one + named after it for any other. root says whether the file roots a target, + which by default the places where Cargo discovers targets say.""" + directory, name = posixpath.split(relative) + if root is None: + root = _target_root(relative) + if name == "mod.rs" or root: + return directory + return posixpath.join(directory, name[: -len(".rs")]) + + +def _declarer_candidates(relative: str) -> list[str]: + """Returns the files, relative to the crate, that may declare the module + of the file relative, nearest first: for each directory that holds it, the + file named after the directory and its mod.rs, main.rs, and lib.rs.""" + directory = posixpath.dirname(relative) + if posixpath.basename(relative) == "mod.rs": + directory = posixpath.dirname(directory) + parts = directory.split("/") if directory else [] + found: list[str] = [] + for cut in range(len(parts), 0, -1): + prefix = "/".join(parts[:cut]) + found += [ + f"{prefix}{suffix}" for suffix in (".rs", "/mod.rs", "/main.rs", "/lib.rs") + ] + return found + + +def _declared( + declarer: str, + inline: Sequence[str], + name: str, + path_attribute: str | None, + root: bool | None = None, +) -> tuple[str, ...]: + """Returns the files, relative to the crate, that may hold the module that + `mod NAME;` declares in declarer, inside the inline modules inline, with + the #[path] path_attribute if it has one; root says whether declarer roots + a target.""" + directory = posixpath.join(_module_directory(declarer, root), *inline) + if path_attribute is not None: + if not inline: + directory = posixpath.dirname(declarer) + return (posixpath.normpath(posixpath.join(directory, path_attribute)),) + return ( + posixpath.join(directory, f"{name}.rs"), + posixpath.join(directory, name, "mod.rs"), + ) + + +def _path_attribute(parsed: RustFile, item: Item) -> str | None: + for attribute in item.attributes: + if attribute.path == "path": + for token in parsed.tokens[attribute.start : attribute.end]: + if token.kind == LITERAL and token.text.startswith('"'): + return token.text[1:-1] + return None + + +def _module_declarations( + parsed: RustFile, +) -> list[tuple[tuple[str, ...], str, str | None, frozenset[str]]]: + """Returns the `mod NAME;` declarations of a file: the inline modules that + hold each, its name, its #[path], and the predicates that it and those + modules require.""" + found: list[tuple[tuple[str, ...], str, str | None, frozenset[str]]] = [] + + def walk( + items: Sequence[Item], inline: tuple[str, ...], held: frozenset[str] + ) -> None: + for item in items: + if item.kind != "mod": + continue + conditions = held | _conjuncts(parsed, item.attributes) + if item.body is None: + path = _path_attribute(parsed, item) + found.append((inline, item.name, path, conditions)) + else: + inner = conditions | _conjuncts(parsed, item.inner_attributes) + walk(item.children, (*inline, item.name), inner) + + walk(parsed.items, (), frozenset()) + return found + + +# A #[path] attribute, which may set the file of a module anywhere. +_PATH_ATTRIBUTE = re.compile(r"#\s*\[\s*path\s*=") + + +def crate_targets( + available: Collection[str], + read: Callable[[Sequence[str]], Mapping[str, str]], + crates: Sequence[str], +) -> dict[str, dict[str, bool]]: + """Returns, for each crate, the roots of its targets, relative to it, each + with whether `cargo test` runs its tests, as target_roots finds them in its + manifest and among available, the paths of a commit's files. read returns + the text of each of the files that it is given.""" + manifests = read( + sorted( + manifest + for crate in crates + if (manifest := _crate_path(crate, "Cargo.toml")) in available + ) + ) + members: dict[str, set[str]] = {crate: set() for crate in crates} + for path in available: + crate = layout.crate_of(path, crates) + if crate is not None: + members[crate].add(layout.crate_relative(path, crate)) + return { + crate: layout.target_roots( + manifests.get(_crate_path(crate, "Cargo.toml"), ""), members[crate] + ) + for crate in crates + } + + +def module_conditions( + available: Collection[str], + read: Callable[[Sequence[str]], Mapping[str, str]], + crates: Sequence[str], + paths: Iterable[str], +) -> dict[str, Conditions | None]: + """Returns, for each Rust file among paths, the ways that its module + compiles: for each declaration of it, through each chain of declarations + of its ancestors, the cfg predicates that those declarations, the inline + modules that hold them, and their files' inner attributes require; or + None for a file that no target of its crate whose tests `cargo test` runs + compiles, since no module that such a target's root, as crate_targets + finds it, reaches declares it. available are the paths of a commit's + files, and read returns the text of each of the files that it is given.""" + paths = list(paths) + targets = crate_targets(available, read, crates) + wanted = { + crate for path in paths if (crate := layout.crate_of(path, crates)) is not None + } + texts = read( + sorted( + path + for path in available + if path.endswith(".rs") and layout.crate_of(path, crates) in wanted + ) + ) + # The files that may declare a module in another directory with #[path]. + redirecting: dict[str, list[str]] = {crate: [] for crate in wanted} + for path, text in sorted(texts.items()): + crate = layout.crate_of(path, crates) + if crate is not None and _PATH_ATTRIBUTE.search(text): + redirecting[crate].append(layout.crate_relative(path, crate)) + + def candidates(crate: str, relative: str) -> list[str]: + """The files that may declare the module of relative: those that its + directories name, the roots of the targets whose modules lie in one of + its directories, and those with a #[path], which may name it from + anywhere.""" + directory = posixpath.dirname(relative) + return [ + *_declarer_candidates(relative), + *( + root + for root, tested in sorted(targets[crate].items()) + if (directory + "/").startswith(_module_directory(root, tested) + "/") + ), + *redirecting[crate], + ] + + read_declarations: dict[ + str, + tuple[ + list[tuple[tuple[str, ...], str, str | None, frozenset[str]]], + frozenset[str], + ], + ] = {} + + def declarations( + path: str, + ) -> tuple[ + list[tuple[tuple[str, ...], str, str | None, frozenset[str]]], frozenset[str] + ]: + if path not in read_declarations: + try: + parsed = layout.parse_rust(texts[path]) + except layout.RustSyntaxError: + read_declarations[path] = ([], frozenset()) + else: + read_declarations[path] = ( + _module_declarations(parsed), + _conjuncts(parsed, parsed.inner_attributes), + ) + return read_declarations[path] + + memo: dict[str, Conditions | None] = {} + + def inherited(path: str) -> Conditions | None: + if path in memo: + return memo[path] + # A module that declares itself, through its ancestors, is reached by + # none of them. + memo[path] = None + crate = layout.crate_of(path, crates) + if crate is None: + return memo[path] + relative = layout.crate_relative(path, crate) + # The root of a target whose tests run compiles; that of another may + # still compile as a module that such a target declares. + if targets[crate].get(relative): + memo[path] = ANYWHERE + return memo[path] + # Every declaration that reaches the module is a way that it compiles. + ways: set[frozenset[str]] = set() + for candidate in candidates(crate, relative): + full = _crate_path(crate, candidate) + if full not in texts or full == path: + continue + found, own = declarations(full) + # A root declares the modules of its directory where its target + # compiles it, which for tests is where it runs them. + root = targets[crate].get(candidate, False) + for inline, name, path_attribute, conditions in found: + if relative in _declared(candidate, inline, name, path_attribute, root): + parent = inherited(full) + if parent is not None: + ways.update(way | own | conditions for way in parent) + memo[path] = _reduced(ways) if ways else None + return memo[path] + + return {path: inherited(path) for path in paths} + + +def _requires_test(conditions: Conditions | None) -> bool: + """Whether code that compiles in the ways that conditions say compiles only + where `test` holds: each way has a predicate that requires it, as the + `test` of a `#[cfg(test)] mod` declaration of its module or of an ancestor + does.""" + + def requires(predicate: str) -> bool: + try: + tokens = layout.tokenize_rust(predicate) + partner = layout.match_brackets(tokens) + except layout.RustSyntaxError: + return False + return layout.requires_test(layout.parse_cfg(tokens, partner, 0, len(tokens))) + + return bool(conditions) and all( + any(requires(predicate) for predicate in way) for way in conditions or () + ) + + +def _entries( + path: str, + blob: str, + data: bytes, + crates: Sequence[str], + kind: tuple[str, str], + inherited: Conditions = ANYWHERE, +) -> list[Entry]: + category, mode = kind + if mode in ("rust", "rust-production"): + crate = layout.crate_of(path, crates) + if crate is not None: + try: + return rust_entries( + path, + crate, + layout.decode(data), + category, + mode == "rust", + inherited, + ) + except layout.RustSyntaxError: + pass + elif mode == "python": + try: + return python_entries(path, layout.decode(data), category) + except (SyntaxError, ValueError, RecursionError, MemoryError): + pass + return [file_entry(path, blob, category)] + + +def _affected( + changed: Collection[str], sides: Sequence[tuple[Tree, Sequence[str]]] +) -> set[str]: + """Returns the Rust files of either commit in the crates whose Rust files + or manifest changed, but the changed files: any of them may compile in + other ways, since a changed declaration reaches the modules beneath it, + and through a #[path] modules anywhere in its crate, and a changed + manifest every target of its crate.""" + touched: set[str] = set() + for path in changed: + for _, crates in sides: + crate = layout.crate_of(path, crates) + if crate is not None and ( + path.endswith(".rs") or path == _crate_path(crate, "Cargo.toml") + ): + touched.add(crate) + return { + path + for tree, crates in sides + for path in tree.blobs + if path.endswith(".rs") + and path not in changed + and layout.crate_of(path, crates) in touched + } + + +def compare_commits(root: Path, base: str, head: str) -> Comparison: + """Compares the test assets of the merge base of base and head with those + of head.""" + head_commit = resolve(root, head) + base_commit = ( + layout.git(root, "merge-base", resolve(root, base), head_commit) + .decode("ascii") + .strip() + ) + before, after = Tree(root, base_commit), Tree(root, head_commit) + changed = sorted( + path + for path in before.blobs.keys() | after.blobs.keys() + if before.blobs.get(path) != after.blobs.get(path) + ) + sides = ( + (before, layout.rust_crates(before.blobs)), + (after, layout.rust_crates(after.blobs)), + ) + + def reader(tree: Tree) -> Callable[[Sequence[str]], Mapping[str, str]]: + def read(paths: Sequence[str]) -> Mapping[str, str]: + blobs = read_blobs(root, (tree.blobs[path] for path in paths)) + return {path: layout.decode(blobs[tree.blobs[path]]) for path in paths} + + return read + + # The ways that the modules of the Rust files compile, which a move must + # keep, and which a changed declaration or manifest may change for any + # unchanged file of its crate. + affected = _affected(set(changed), sides) + inherited = [ + module_conditions( + tree.blobs.keys(), + reader(tree), + crates, + ( + path + for path in (*changed, *affected) + if path in tree.blobs + and (kind := classify(path, crates)) is not None + and kind[1] in ("rust", "rust-production") + ), + ) + for tree, crates in sides + ] + changed = sorted( + { + *changed, + *( + path + for path in affected + if path in inherited[0] + and path in inherited[1] + and inherited[0][path] != inherited[1][path] + ), + } + ) + kinds = [ + {path: kind for path in changed if (kind := classify(path, crates)) is not None} + for _, crates in sides + ] + # A production file whose module only tests compile holds test code alone. + for (tree, _), found, conditions in zip(sides, kinds, inherited, strict=True): + test_only = [ + path + for path, (_, mode) in found.items() + if mode == "rust-production" + and path in tree.blobs + and _requires_test(conditions.get(path)) + ] + for path in test_only: + found[path] = (TEST_CODE, "rust") + data = read_blobs( + root, + ( + tree.blobs[path] + for (tree, _), found in zip(sides, kinds, strict=True) + for path in found + if path in tree.blobs + ), + ) + + def text(side: int, path: str) -> str | None: + tree = sides[side][0] + if path not in tree.blobs: + return None + return layout.decode(data[tree.blobs[path]]) + + comparison = Comparison(base_commit, head_commit) + entries: list[list[Entry]] = [[], []] + for path in changed: + kind = kinds[1].get(path) or kinds[0].get(path) + if kind is None: + continue + mode = kind[1] + if mode == "allowlist": + comparison.changes.extend(allowlist_changes(text(0, path), text(1, path))) + elif mode == "lines": + comparison.changes.extend(line_changes(path, text(0, path), text(1, path))) + elif mode == "tokens": + comparison.changes.extend(token_changes(path, text(0, path), text(1, path))) + comparison.changes.extend(job_changes(path, text(0, path), text(1, path))) + elif mode == "shards": + comparison.changes.extend(shard_changes(text(0, path), text(1, path))) + elif mode == "file" and kind[0] == TEST_POLICY: + if path in before.blobs: + comparison.changes.append( + Change( + TEST_POLICY, + "modified" if path in after.blobs else "removed", + "file", + f"`{path}`", + Location(path, 0), + Location(path, 0) if path in after.blobs else None, + ) + ) + else: + for side, (tree, crates) in enumerate(sides): + found = kinds[side].get(path) + if found is None or path not in tree.blobs: + continue + conditions = inherited[side].get(path, ANYWHERE) + if conditions is None: + # No target compiles the file, so its tests never run. + continue + blob = tree.blobs[path] + entries[side].extend( + _entries(path, blob, data[blob], crates, found, conditions) + ) + comparison.changes.extend(compare(entries[0], entries[1])) + comparison.changes.sort(key=Change.order) + return comparison + + +# Reports. + + +def _link(repository: str | None, commit: str, location: Location) -> str: + label = f"{location.path}:{location.line}" if location.line else location.path + label = label.replace("`", "'").replace("|", "\\|") + if repository is None: + return f"`{label}`" + url = ( + f"https://github.com/{repository}/blob/{commit}/" + f"{urllib.parse.quote(location.path)}" + ) + if location.line: + url += f"#L{location.line}" + return f"[`{label}`]({url})" + + +def _cell(text: str) -> str: + return text.replace("|", "\\|").replace("\n", " ").replace("@", "@\u200b") + + +def render( + comparison: Comparison, repository: str | None, rows: int = ROWS_PER_SECTION +) -> str: + """Renders the comparison as the Markdown of the pull request's comment.""" + flagged = comparison.flagged + lines = [COMMENT_MARKER] + guide = "doc/contribute.md#changes-to-existing-tests" + if repository is not None: + guide = f"[{guide}](https://github.com/{repository}/blob/dev/{guide})" + if flagged: + lines += [ + "### This pull request changes existing tests", + "", + f"It modifies or removes {len(flagged)} existing test asset(s), " + "listed below. Review each with care: a change to a test can hide " + f"a regression that the test caught. See {guide}.", + ] + else: + lines += ["### This pull request changes no existing test"] + for category in CATEGORIES: + found = [change for change in flagged if change.category == category] + if not found: + continue + lines += ["", f"#### {category[0].upper()}{category[1:]}", ""] + lines += ["| Change | What | Before | After |", "| --- | --- | --- | --- |"] + for change in found[:rows]: + action = f"{change.action} {change.kind}" + what = change.what + (f" ({change.detail})" if change.detail else "") + before = ( + _link(repository, comparison.base, change.base) if change.base else "" + ) + after = ( + _link(repository, comparison.head, change.head) if change.head else "" + ) + lines.append(f"| {_cell(action)} | {_cell(what)} | {before} | {after} |") + if len(found) > rows: + lines += [ + "", + f"... and {len(found) - rows} more, which the job's summary lists.", + ] + lines += [ + "", + f"Relocated unchanged: {comparison.count('relocated')}; added: " + f"{comparison.count('added')}. Compared the merge base " + f"`{comparison.base[:12]}` with `{comparison.head[:12]}`.", + ] + return "\n".join(lines) + "\n" + + +def comment_body(comparison: Comparison, repository: str | None) -> str: + """Renders the comment, with fewer rows per section until it fits.""" + rows = ROWS_PER_SECTION + while True: + body = render(comparison, repository, rows) + if len(body) <= COMMENT_LIMIT or rows <= 1: + return body[:COMMENT_LIMIT] + rows //= 2 + + +def summary_lines(comparison: Comparison) -> list[str]: + """Renders the flagged changes as plain text, one per line.""" + lines: list[str] = [] + for change in comparison.flagged: + where = " -> ".join( + f"{location.path}:{location.line}" if location.line else location.path + for location in (change.base, change.head) + if location is not None + ) + detail = f" ({change.detail})" if change.detail else "" + lines.append( + f"{change.category}: {change.action} {change.kind} " + f"{change.what.replace('`', '')}{detail} [{where}]" + ) + return lines + + +# Publication. + +Runner = Callable[[Sequence[str], str | None], subprocess.CompletedProcess[str]] + + +def _run( + arguments: Sequence[str], stdin: str | None +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + list(arguments), input=stdin, capture_output=True, text=True, check=False + ) + + +class GitHub: + """The calls to GitHub that flag a pull request, through the gh CLI.""" + + def __init__(self, repository: str, run: Runner = _run) -> None: + self.repository = repository + self.run = run + + def api( + self, + endpoint: str, + *, + method: str = "GET", + body: Mapping[str, object] | None = None, + options: Sequence[str] = (), + tolerate: Sequence[int] = (), + ) -> str | None: + """Calls the REST API; returns None for an HTTP status that tolerate + lists, and raises FlagError for any other failure.""" + arguments = ["gh", "api", "--method", method, *options, endpoint] + stdin = None + if body is not None: + arguments += ["--input", "-"] + stdin = json.dumps(body) + result = self.run(arguments, stdin) + if result.returncode != 0: + if any(f"HTTP {status}" in result.stderr for status in tolerate): + return None + raise FlagError( + f"gh api {method} {endpoint} failed: {result.stderr.strip()}" + ) + return result.stdout + + def publish( + self, number: int, body: str, flagged: bool, label: str = LABEL + ) -> None: + """Adds or removes the label, and writes or resolves the comment.""" + repository = self.repository + name = urllib.parse.quote(label, safe="") + if flagged: + if self.api(f"repos/{repository}/labels/{name}", tolerate=(404,)) is None: + # Another run may create the label first. + self.api( + f"repos/{repository}/labels", + method="POST", + body={ + "name": label, + "color": LABEL_COLOR, + "description": LABEL_DESCRIPTION, + }, + tolerate=(422,), + ) + self.api( + f"repos/{repository}/issues/{number}/labels", + method="POST", + body={"labels": [label]}, + ) + else: + self.api( + f"repos/{repository}/issues/{number}/labels/{name}", + method="DELETE", + tolerate=(404,), + ) + listing = self.api( + f"repos/{repository}/issues/{number}/comments", + options=( + "--paginate", + "--jq", + '.[] | select(.user.login == "github-actions[bot]" and ' + f'(.body | startswith("{COMMENT_MARKER}"))) | .id', + ), + ) + comments = [line for line in (listing or "").split() if line.isdigit()] + if comments: + self.api( + f"repos/{repository}/issues/comments/{comments[0]}", + method="PATCH", + body={"body": body}, + ) + elif flagged: + self.api( + f"repos/{repository}/issues/{number}/comments", + method="POST", + body={"body": body}, + ) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Flag the changes of a pull request to existing tests." + ) + parser.add_argument("--root", type=Path, default=REPO_ROOT, help="Git repository") + parser.add_argument( + "--base", + default=DEFAULT_BASE, + help=f"the pull request's base; its merge base with head counts (default: {DEFAULT_BASE})", + ) + parser.add_argument("--head", default="HEAD", help="the pull request's head") + parser.add_argument("--repository", help="OWNER/REPO, for links and --publish") + parser.add_argument("--report", type=Path, help="write the comment's Markdown here") + parser.add_argument("--summary", type=Path, help="append the full report here") + parser.add_argument( + "--github-output", type=Path, help="append flagged= and count= here" + ) + parser.add_argument( + "--publish", + type=int, + metavar="NUMBER", + help="label and comment on this pull request with gh; needs --repository", + ) + parser.add_argument("--label", default=LABEL, help=f"the label (default: {LABEL})") + args = parser.parse_args(argv) + if args.publish is not None and not args.repository: + parser.error("--publish needs --repository") + try: + comparison = compare_commits(args.root, args.base, args.head) + flagged = comparison.flagged + body = comment_body(comparison, args.repository) + if args.report is not None: + args.report.write_text(body, encoding="utf-8") + if args.summary is not None: + with args.summary.open("a", encoding="utf-8") as summary: + summary.write(render(comparison, args.repository, len(flagged) or 1)) + if args.github_output is not None: + with args.github_output.open("a", encoding="utf-8") as output: + output.write(f"flagged={'true' if flagged else 'false'}\n") + output.write(f"count={len(flagged)}\n") + if args.publish is not None: + GitHub(args.repository).publish( + args.publish, body, bool(flagged), args.label + ) + except (layout.CheckError, FlagError, OSError) as error: + print(f"error: {error}", file=sys.stderr) + return 2 + for line in summary_lines(comparison): + print(line) + print( + f"{len(flagged)} change(s) to existing tests; relocated unchanged: " + f"{comparison.count('relocated')}; added: {comparison.count('added')}" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/check_test_layout.py b/scripts/check_test_layout.py new file mode 100644 index 000000000..39d7a48d7 --- /dev/null +++ b/scripts/check_test_layout.py @@ -0,0 +1,2047 @@ +#!/usr/bin/env python3 +"""Check that tests live apart from the code that they test. + +A Rust crate of the repository, a directory with a tracked Cargo.toml, keeps +its tests in test files: the files beneath the crate's `tests` directory, and +in `src/`, the `tests.rs` files and the files beneath a `tests` directory. The +crate's other files in `src/` are production files. A production file +declares its unit tests only as an out-of-line module named `tests` whose cfg +requires `test`, and whose file holds them: + + #[cfg(test)] + mod tests; + +A test file belongs to a module that a target of its crate whose tests +`cargo test` runs, its library, a binary, or an integration test, as the +crate's manifest names it or Cargo discovers it, reaches through `mod` +declarations from its root that builds of tests compile: no cfg that no build +of tests satisfies, such as `any()` or `not(test)`, guards one. The Rust +files of `tests/fixtures/` and `tests/vectors/` are data, not modules. + +A production file holds no other test code: outside function bodies, no +inline module, module of another name, or item that a cfg requiring `test` +guards, directly or through cfg_attr; no test attribute, such as #[test], #[should_panic], or #[ignore]; no +proptest! block; and no #![cfg(test)] that makes the whole file test-only. +What it may hold are test hooks, the attributes and cfg! invocations whose +predicates mention `test`, such as a field, statement, or match arm that only +tests compile, or a derive that only tests need: exactly as many as the seams +of scripts/test-layout-allowlist.json allow the file. A hook is test code that +cannot move out of the code that it hooks. + +The production files that predate these rules keep their test code until it +moves out: the baseline of the allowlist lists each with the numbers of test +hooks and of tests that it holds, which may only go down. `--tighten` lowers +the allowlist to what the files hold, and moves to the seams a baseline file +that holds no test code but hooks. It never raises an entry or adds a file. + +Every module named `tests` is out of line and requires `test`, its file +exists, and every test file in `src/` belongs to a module that a file +declares. No file in `src/` sets a module's file with #[path]. + +Python unit tests are the `test_*.py` modules of scripts/, +.github/specula/tests/, and .github/skills/*/scripts/, and the end-to-end +scenario drivers are the `*_tests.py` modules of scripts/nvx_tools/. No other +module imports unittest, pytest, doctest, a `test_*` module, or the +PowerShell test support of powershell_testing.py. + +The checks read the files that Git tracks, so stage a new file before you run +them. +""" + +from __future__ import annotations + +import argparse +import ast +import bisect +import json +import os +import posixpath +import re +import subprocess +import sys +from collections.abc import Collection, Iterable, Mapping, Sequence +from dataclasses import dataclass, field +from pathlib import Path +from typing import Protocol, cast + +REPO_ROOT = Path(__file__).resolve().parents[1] +ALLOWLIST = "scripts/test-layout-allowlist.json" +ALLOWLIST_SCHEMA = "nvx-test-layout-allowlist" +ALLOWLIST_SCHEMA_VERSION = 1 +TIGHTEN_COMMAND = "python3 scripts/check_test_layout.py --tighten" + +# The directories, relative to a crate, of the data of its tests, such as +# fixtures and conformance vectors, whose Rust files are no modules. +TEST_DATA_DIRECTORIES = ("tests/fixtures/", "tests/vectors/") + +# The places of the Python unit tests and of the end-to-end scenario drivers, +# and the modules that only the unit tests import. +PYTHON_UNIT_TESTS = re.compile( + r"(?:scripts|\.github/specula/tests|\.github/skills/[^/]+/scripts)" + r"/test_[^/]*\.py" +) +PYTHON_END_TO_END_TESTS = re.compile(r"scripts/nvx_tools/[^/]*_tests\.py") +PYTHON_TEST_SUPPORT = frozenset({"scripts/powershell_testing.py"}) +PYTHON_TEST_FRAMEWORKS = frozenset({"doctest", "pytest", "unittest"}) + +# Attributes that only tests take, and those of them that make a function a +# test. +TEST_ATTRIBUTES = frozenset({"bench", "ignore", "should_panic", "test"}) +TEST_FUNCTION_ATTRIBUTES = frozenset({"bench", "test"}) + +IDENT = "ident" +LIFETIME = "lifetime" +LITERAL = "literal" +PUNCT = "punct" + + +class CheckError(RuntimeError): + """Raised when the files of the repository cannot be listed or read.""" + + +@dataclass(frozen=True, order=True) +class Problem: + """A place where test code is not where the layout puts it.""" + + path: str + line: int + rule: str + message: str + + def __str__(self) -> str: + location = f"{self.path}:{self.line}" if self.line else self.path + return f"{location}: {self.rule}: {self.message}" + + def annotation(self) -> str: + """Formats the problem as a GitHub Actions error annotation.""" + properties = f"file={_escape_property(self.path)}" + if self.line: + properties += f",line={self.line}" + return f"::error {properties}::{_escape_data(str(self))}" + + +def _escape_data(value: str) -> str: + return value.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") + + +def _escape_property(value: str) -> str: + return _escape_data(value).replace(":", "%3A").replace(",", "%2C") + + +# Rust tokens. + + +class RustSyntaxError(ValueError): + """Raised when a Rust file does not split into tokens whose brackets + balance.""" + + def __init__(self, line: int, message: str) -> None: + super().__init__(f"line {line}: {message}") + self.line = line + + +@dataclass(frozen=True) +class Token: + """A token of Rust source: an identifier, which includes keywords and raw + identifiers; a lifetime or label; a literal; or one punctuation + character. Comments, doc comments included, are no tokens.""" + + kind: str + text: str + start: int + end: int + line: int + + +_SPACE_OR_LINE_COMMENT = re.compile(r"(?:\s+|//[^\n]*)+") +_BLOCK_COMMENT_DELIMITER = re.compile(r"/\*|\*/") +_IDENTIFIER = re.compile(r"[^\W\d]\w*") +_NUMBER = re.compile(r"[0-9]\w*(?:\.[0-9]\w*)?") +_STRING_REST = re.compile(r'[^"\\]*(?:\\.[^"\\]*)*"', re.DOTALL) +_CHAR = re.compile( + r"'(?:[^'\\\n]|\\(?:u\{[0-9A-Fa-f_]*\}|x[0-9A-Fa-f]{2}|.))'", re.DOTALL +) +_LIFETIME = re.compile(r"'(?:r#)?[^\W\d]\w*") + + +def tokenize_rust(text: str) -> list[Token]: + """Splits Rust source into tokens, without its comments.""" + newlines = [match.start() for match in re.finditer("\n", text)] + + def line_of(offset: int) -> int: + return bisect.bisect_right(newlines, offset) + 1 + + tokens: list[Token] = [] + + def add(kind: str, start: int, end: int) -> int: + tokens.append(Token(kind, text[start:end], start, end, line_of(start))) + return end + + position = 1 if text.startswith("\ufeff") else 0 + size = len(text) + while position < size: + skipped = _SPACE_OR_LINE_COMMENT.match(text, position) + if skipped is not None: + position = skipped.end() + continue + if text.startswith("/*", position): + position = _block_comment_end(text, position, line_of(position)) + continue + char = text[position] + if char == '"': + position = add( + LITERAL, position, _string_end(text, position, position + 1, line_of) + ) + continue + if char == "'": + literal = _CHAR.match(text, position) + if literal is not None: + position = add(LITERAL, position, literal.end()) + continue + lifetime = _LIFETIME.match(text, position) + if lifetime is not None: + position = add(LIFETIME, position, lifetime.end()) + continue + position = add(PUNCT, position, position + 1) + continue + if "0" <= char <= "9": + number = _NUMBER.match(text, position) + assert number is not None + position = add(LITERAL, position, number.end()) + continue + word = _IDENTIFIER.match(text, position) + if word is None: + position = add(PUNCT, position, position + 1) + continue + end = word.end() + prefix = word.group() + following = text[end] if end < size else "" + if prefix in ("r", "br", "cr") and following in ('"', "#"): + hashes = end + while hashes < size and text[hashes] == "#": + hashes += 1 + if hashes < size and text[hashes] == '"': + delimiter = '"' + "#" * (hashes - end) + close = text.find(delimiter, hashes + 1) + if close < 0: + raise RustSyntaxError(line_of(position), "unterminated raw string") + position = add(LITERAL, position, close + len(delimiter)) + continue + if prefix == "r" and hashes == end + 1: + raw = _IDENTIFIER.match(text, hashes) + if raw is not None: + position = add(IDENT, position, raw.end()) + continue + elif prefix in ("b", "c") and following == '"': + position = add( + LITERAL, position, _string_end(text, position, end + 1, line_of) + ) + continue + elif prefix == "b" and following == "'": + literal = _CHAR.match(text, end) + if literal is not None: + position = add(LITERAL, position, literal.end()) + continue + position = add(IDENT, position, end) + return tokens + + +def _block_comment_end(text: str, start: int, line: int) -> int: + depth = 0 + position = start + while True: + delimiter = _BLOCK_COMMENT_DELIMITER.search(text, position) + if delimiter is None: + raise RustSyntaxError(line, "unterminated block comment") + depth += 1 if delimiter.group() == "/*" else -1 + position = delimiter.end() + if depth == 0: + return position + + +def _string_end(text: str, start: int, body: int, line_of: _LineOf) -> int: + rest = _STRING_REST.match(text, body) + if rest is None: + raise RustSyntaxError(line_of(start), "unterminated string") + return rest.end() + + +class _LineOf(Protocol): + def __call__(self, offset: int, /) -> int: ... + + +_CLOSERS = {")": "(", "]": "[", "}": "{"} + + +def match_brackets(tokens: Sequence[Token]) -> list[int]: + """Returns, for each token, the index of the bracket that it opens or + closes, or -1.""" + partner = [-1] * len(tokens) + stack: list[int] = [] + for index, token in enumerate(tokens): + if token.kind != PUNCT: + continue + if token.text in "([{": + stack.append(index) + elif token.text in _CLOSERS: + if not stack or tokens[stack[-1]].text != _CLOSERS[token.text]: + raise RustSyntaxError(token.line, f"unbalanced {token.text!r}") + opener = stack.pop() + partner[opener] = index + partner[index] = opener + if stack: + raise RustSyntaxError(tokens[stack[-1]].line, "unclosed bracket") + return partner + + +def unraw(name: str) -> str: + """Returns an identifier without its raw prefix, `r#`.""" + return name[2:] if name.startswith("r#") else name + + +def is_token(tokens: Sequence[Token], index: int, kind: str, *texts: str) -> bool: + """Whether the token at index is of kind and, if any texts are given, one + of them.""" + if index >= len(tokens): + return False + token = tokens[index] + return token.kind == kind and (not texts or token.text in texts) + + +def _path(tokens: Sequence[Token], start: int, end: int) -> tuple[str, int]: + """Reads a path of identifiers, such as `tokio::test`, from start; returns + it, empty when none starts there, and the index after it.""" + index = start + if is_token(tokens, index, PUNCT, ":") and is_token(tokens, index + 1, PUNCT, ":"): + index += 2 + parts: list[str] = [] + while index < end and is_token(tokens, index, IDENT): + parts.append(unraw(tokens[index].text)) + index += 1 + if ( + index + 2 < end + and is_token(tokens, index, PUNCT, ":") + and is_token(tokens, index + 1, PUNCT, ":") + and is_token(tokens, index + 2, IDENT) + ): + index += 2 + continue + break + if not parts: + return "", start + return "::".join(parts), index + + +def split_commas( + tokens: Sequence[Token], partner: Sequence[int], start: int, end: int +) -> list[tuple[int, int]]: + """Splits the tokens from start to end at the commas outside brackets.""" + pieces: list[tuple[int, int]] = [] + piece = start + index = start + while index < end: + token = tokens[index] + if token.kind == PUNCT: + if token.text in "([{": + index = partner[index] + 1 + continue + if token.text == ",": + pieces.append((piece, index)) + piece = index + 1 + index += 1 + pieces.append((piece, end)) + return [(first, last) for first, last in pieces if first < last] + + +# cfg predicates and attributes. + + +@dataclass(frozen=True) +class Cfg: + """A cfg predicate: `all`, `any`, or `not` of operands; an `option`, with + the literal of its value if it has one; or `other`, whatever does not + parse, with its tokens.""" + + operator: str + name: str = "" + value: str | None = None + operands: tuple[Cfg, ...] = () + words: tuple[str, ...] = () + + +def parse_cfg( + tokens: Sequence[Token], partner: Sequence[int], start: int, end: int +) -> Cfg: + """Parses the cfg predicate from start to end.""" + if start < end and tokens[start].kind == IDENT: + name = unraw(tokens[start].text) + if start + 1 == end: + return Cfg("option", name) + if ( + name in ("all", "any", "not") + and is_token(tokens, start + 1, PUNCT, "(") + and partner[start + 1] == end - 1 + ): + operands = tuple( + parse_cfg(tokens, partner, first, last) + for first, last in split_commas(tokens, partner, start + 2, end - 1) + ) + return Cfg(name, operands=operands) + if ( + start + 3 == end + and is_token(tokens, start + 1, PUNCT, "=") + and is_token(tokens, start + 2, LITERAL) + ): + return Cfg("option", name, tokens[start + 2].text) + words = tuple( + token.text + for index, token in enumerate(tokens[start:end], start) + if token.kind == IDENT + and unraw(token.text) == "test" + and not is_token(tokens, index - 1, PUNCT, "$") + and not is_token(tokens, index + 1, PUNCT, "=") + ) + return Cfg("other", words=words) + + +def mentions_test(cfg: Cfg) -> bool: + """Whether the predicate depends on the `test` option.""" + if cfg.operator == "option": + return cfg.name == "test" and cfg.value is None + if cfg.operator == "other": + return bool(cfg.words) + return any(mentions_test(operand) for operand in cfg.operands) + + +def requires_test(cfg: Cfg) -> bool: + """Whether the predicate holds only when `test` does.""" + if cfg.operator == "option": + return cfg.name == "test" and cfg.value is None + if cfg.operator == "all": + return any(requires_test(operand) for operand in cfg.operands) + if cfg.operator == "any": + live = [operand for operand in cfg.operands if not _never(operand)] + return bool(live) and all(requires_test(operand) for operand in live) + if cfg.operator == "not": + return len(cfg.operands) == 1 and forbids_test(cfg.operands[0]) + return False + + +def forbids_test(cfg: Cfg) -> bool: + """Whether the predicate holds only when `test` does not.""" + if cfg.operator == "all": + return any(forbids_test(operand) for operand in cfg.operands) + if cfg.operator == "any": + live = [operand for operand in cfg.operands if not _never(operand)] + return bool(live) and all(forbids_test(operand) for operand in live) + if cfg.operator == "not": + return len(cfg.operands) == 1 and requires_test(cfg.operands[0]) + return False + + +def _never(cfg: Cfg) -> bool: + """Whether the predicate holds for no configuration, as `any()` does.""" + if cfg.operator == "any": + return all(_never(operand) for operand in cfg.operands) + if cfg.operator == "all": + return any(_never(operand) for operand in cfg.operands) + if cfg.operator == "not": + return len(cfg.operands) == 1 and _always(cfg.operands[0]) + return False + + +def _always(cfg: Cfg) -> bool: + """Whether the predicate holds for every configuration, as `all()` does.""" + if cfg.operator == "all": + return all(_always(operand) for operand in cfg.operands) + if cfg.operator == "any": + return any(_always(operand) for operand in cfg.operands) + if cfg.operator == "not": + return len(cfg.operands) == 1 and _never(cfg.operands[0]) + return False + + +def excludes_tests(cfg: Cfg) -> bool: + """Whether the predicate holds in no build of tests: for no configuration, + as `any()`, or only where `test` does not, as `not(test)`.""" + return _never(cfg) or forbids_test(cfg) + + +@dataclass(frozen=True) +class Attribute: + """An attribute: the indices of its `#` and of its closing `]`, its path, + such as `cfg` or `tokio::test`, empty when it starts with none, and the + range of the tokens between the delimiters after its path, if any.""" + + start: int + end: int + inner: bool + path: str + arguments: tuple[int, int] | None + line: int + + +def attribute_at( + tokens: Sequence[Token], partner: Sequence[int], index: int +) -> Attribute | None: + """Returns the attribute whose `#` is at index, if one is.""" + if not is_token(tokens, index, PUNCT, "#"): + return None + bracket = index + 1 + inner = is_token(tokens, bracket, PUNCT, "!") + if inner: + bracket += 1 + if not is_token(tokens, bracket, PUNCT, "["): + return None + close = partner[bracket] + path, after = _path(tokens, bracket + 1, close) + arguments = None + if path and after < close and is_token(tokens, after, PUNCT, "(", "[", "{"): + arguments = (after + 1, partner[after]) + return Attribute(index, close, inner, path, arguments, tokens[index].line) + + +def attribute_cfg( + tokens: Sequence[Token], partner: Sequence[int], attribute: Attribute +) -> Cfg | None: + """Returns the predicate of a #[cfg(...)] attribute.""" + if attribute.path != "cfg" or attribute.arguments is None: + return None + return parse_cfg(tokens, partner, *attribute.arguments) + + +def attribute_requires_test( + tokens: Sequence[Token], partner: Sequence[int], attribute: Attribute +) -> bool: + """Whether the attribute is a cfg that only lets tests compile what it + guards, or a cfg_attr that applies such a cfg wherever its predicate + holds: `cfg_attr(P, cfg(X))` guards with `any(not(P), X)`.""" + if attribute.arguments is None: + return False + if attribute.path == "cfg": + return requires_test(parse_cfg(tokens, partner, *attribute.arguments)) + if attribute.path == "cfg_attr": + return any( + requires_test(cfg) + for cfg in _cfg_attr_guards(tokens, partner, *attribute.arguments, ()) + ) + return False + + +def attribute_excludes_tests( + tokens: Sequence[Token], partner: Sequence[int], attribute: Attribute +) -> bool: + """Whether the attribute is a cfg that no build of tests satisfies, or a + cfg_attr that applies one wherever tests build, so that what it is on + never compiles for tests, as with #[cfg(any())] or + #[cfg_attr(test, cfg(not(test)))].""" + if attribute.arguments is None: + return False + if attribute.path == "cfg": + return excludes_tests(parse_cfg(tokens, partner, *attribute.arguments)) + if attribute.path == "cfg_attr": + return any( + excludes_tests(cfg) + for cfg in _cfg_attr_guards(tokens, partner, *attribute.arguments, ()) + ) + return False + + +def _cfg_attr_guards( + tokens: Sequence[Token], + partner: Sequence[int], + start: int, + end: int, + held: tuple[Cfg, ...], +) -> list[Cfg]: + """Returns the predicates with which the cfgs that the cfg_attr from start + to end applies, where held holds too, guard what it is on.""" + pieces = split_commas(tokens, partner, start, end) + if not pieces: + return [] + held = (*held, parse_cfg(tokens, partner, *pieces[0])) + where = held[0] if len(held) == 1 else Cfg("all", operands=held) + guards: list[Cfg] = [] + for first, last in pieces[1:]: + path, after = _path(tokens, first, last) + if not is_token(tokens, after, PUNCT, "(") or partner[after] != last - 1: + continue + if path == "cfg": + applied = parse_cfg(tokens, partner, after + 1, partner[after]) + guards.append(Cfg("any", operands=(Cfg("not", operands=(where,)), applied))) + elif path == "cfg_attr": + guards.extend( + _cfg_attr_guards(tokens, partner, after + 1, partner[after], held) + ) + return guards + + +def attribute_mentions_test( + tokens: Sequence[Token], partner: Sequence[int], attribute: Attribute +) -> bool: + """Whether the attribute is a cfg or cfg_attr whose predicate, or that of + an attribute that it applies, depends on `test`.""" + if attribute.arguments is None: + return False + if attribute.path == "cfg": + return mentions_test(parse_cfg(tokens, partner, *attribute.arguments)) + if attribute.path == "cfg_attr": + return _cfg_attr_mentions_test(tokens, partner, *attribute.arguments) + return False + + +def _cfg_attr_mentions_test( + tokens: Sequence[Token], partner: Sequence[int], start: int, end: int +) -> bool: + pieces = split_commas(tokens, partner, start, end) + if not pieces: + return False + if mentions_test(parse_cfg(tokens, partner, *pieces[0])): + return True + for first, last in pieces[1:]: + path, after = _path(tokens, first, last) + if not is_token(tokens, after, PUNCT, "(") or partner[after] != last - 1: + continue + if path == "cfg" and mentions_test( + parse_cfg(tokens, partner, after + 1, last - 1) + ): + return True + if path == "cfg_attr" and _cfg_attr_mentions_test( + tokens, partner, after + 1, last - 1 + ): + return True + return False + + +def attribute_paths( + tokens: Sequence[Token], partner: Sequence[int], attribute: Attribute +) -> list[str]: + """Returns the paths of the attribute and of those that it applies through + cfg_attr.""" + paths = [attribute.path] + if attribute.path == "cfg_attr" and attribute.arguments is not None: + paths.extend(_cfg_attr_paths(tokens, partner, *attribute.arguments)) + return paths + + +def _cfg_attr_paths( + tokens: Sequence[Token], partner: Sequence[int], start: int, end: int +) -> list[str]: + paths: list[str] = [] + for first, last in split_commas(tokens, partner, start, end)[1:]: + path, after = _path(tokens, first, last) + paths.append(path) + if path == "cfg_attr" and is_token(tokens, after, PUNCT, "("): + paths.extend(_cfg_attr_paths(tokens, partner, after + 1, partner[after])) + return paths + + +def is_test_attribute(path: str) -> bool: + """Whether an attribute's path makes a test, such as #[test] or + #[tokio::test], or applies only to tests, such as #[ignore].""" + return path in TEST_ATTRIBUTES or path.endswith("::test") + + +def is_test_function_attribute(path: str) -> bool: + """Whether an attribute's path makes its function a test or benchmark.""" + return path in TEST_FUNCTION_ATTRIBUTES or path.endswith("::test") + + +# Rust items. + + +@dataclass +class Item: + """An item: a module, function, type, impl, use, macro invocation, or + another. Its tokens run from start, its first attribute, through end, + exclusive; head is the first token after its attributes. A module, impl, + trait, extern block, or proptest! block holds the items between the braces + of its body.""" + + kind: str + name: str + start: int + head: int + end: int + line: int + attributes: tuple[Attribute, ...] + body: tuple[int, int] | None = None + inner_attributes: tuple[Attribute, ...] = () + children: list[Item] = field(default_factory=list["Item"]) + + +_QUALIFIERS = frozenset({"async", "auto", "default", "gen", "safe", "unsafe"}) +_FUNCTION_QUALIFIERS = frozenset({"async", "extern", "fn", "safe", "unsafe"}) + + +class _ItemParser: + def __init__(self, tokens: Sequence[Token], partner: Sequence[int]) -> None: + self.tokens = tokens + self.partner = partner + + def block(self, start: int, end: int) -> tuple[tuple[Attribute, ...], list[Item]]: + """Parses the inner attributes and the items from start to end.""" + inner: list[Attribute] = [] + index = start + while index < end: + attribute = attribute_at(self.tokens, self.partner, index) + if attribute is None or not attribute.inner: + break + inner.append(attribute) + index = attribute.end + 1 + items: list[Item] = [] + while index < end: + if is_token(self.tokens, index, PUNCT, ";"): + index += 1 + continue + item = self.item(index, end) + items.append(item) + index = item.end + return tuple(inner), items + + def item(self, start: int, end: int) -> Item: + tokens = self.tokens + index = start + attributes: list[Attribute] = [] + while index < end: + attribute = attribute_at(tokens, self.partner, index) + if attribute is None: + break + attributes.append(attribute) + index = attribute.end + 1 + head = index + line = tokens[head].line if head < end else tokens[start].line + + def make( + kind: str, + name: str, + stop: int, + body: tuple[int, int] | None = None, + children: bool = False, + ) -> Item: + item = Item( + kind, name, start, head, max(stop, head + 1), line, tuple(attributes) + ) + if body is not None: + item.body = body + if children: + item.inner_attributes, item.children = self.block( + body[0] + 1, body[1] + ) + return item + + if head >= end: + return Item("other", "", start, head, end, line, tuple(attributes)) + if is_token(tokens, index, IDENT, "pub"): + index += 1 + if is_token(tokens, index, PUNCT, "("): + index = self.partner[index] + 1 + keyword = index + while keyword < end and is_token(tokens, keyword, IDENT): + word = tokens[keyword].text + if word in _QUALIFIERS and is_token(tokens, keyword + 1, IDENT): + keyword += 1 + elif word == "const" and is_token( + tokens, keyword + 1, IDENT, *_FUNCTION_QUALIFIERS + ): + keyword += 1 + elif word == "extern" and not is_token(tokens, keyword + 1, IDENT, "crate"): + after = keyword + 1 + if is_token(tokens, after, LITERAL): + after += 1 + if is_token(tokens, after, PUNCT, "{"): + return make( + "extern", + "", + self.partner[after] + 1, + (after, self.partner[after]), + True, + ) + keyword = after + else: + break + word = tokens[keyword].text if is_token(tokens, keyword, IDENT) else "" + name = ( + unraw(tokens[keyword + 1].text) + if is_token(tokens, keyword + 1, IDENT) + else "" + ) + if word == "extern" and is_token(tokens, keyword + 1, IDENT, "crate"): + name = ( + unraw(tokens[keyword + 2].text) + if is_token(tokens, keyword + 2, IDENT) + else "" + ) + return make("extern crate", name, self._scan(keyword, end, False)[0]) + if word == "mod" and name: + if is_token(tokens, keyword + 2, PUNCT, ";"): + return make("mod", name, keyword + 3) + if is_token(tokens, keyword + 2, PUNCT, "{"): + close = self.partner[keyword + 2] + return make("mod", name, close + 1, (keyword + 2, close), True) + if word in ("fn", "struct", "enum", "trait") or (word == "union" and name): + stop, brace = self._scan(keyword + 1, end, True, True) + if word == "trait" and brace is not None: + return make("trait", name, stop, (brace, stop - 1), True) + return make(word, name, stop, None if brace is None else (brace, stop - 1)) + if word == "impl": + stop, brace = self._scan(keyword + 1, end, True, True) + header = " ".join( + token.text + for token in tokens[keyword : (brace if brace is not None else stop)] + ) + if brace is None: + return make("impl", header, stop) + return make("impl", header, stop, (brace, stop - 1), True) + if word in ("type", "const", "static", "use"): + if word == "static" and is_token(tokens, keyword + 1, IDENT, "mut"): + name = ( + unraw(tokens[keyword + 2].text) + if is_token(tokens, keyword + 2, IDENT) + else "" + ) + return make( + word, "" if word == "use" else name, self._scan(keyword, end, False)[0] + ) + path, after = _path(tokens, keyword, end) + if path and is_token(tokens, after, PUNCT, "!"): + group = after + 1 + if path == "macro_rules" and is_token(tokens, group, IDENT): + name = unraw(tokens[group].text) + group += 1 + else: + name = path + if is_token(tokens, group, PUNCT, "(", "[", "{"): + close = self.partner[group] + stop = close + 1 + brace = tokens[group].text == "{" + if not brace and is_token(tokens, stop, PUNCT, ";"): + stop += 1 + kind = "macro_rules" if path == "macro_rules" else "macro" + if brace and path.split("::")[-1] == "proptest": + return make(kind, name, stop, (group, close), True) + return make(kind, name, stop, (group, close) if brace else None) + return make("other", "", self._scan(head, end, True)[0]) + + def _scan( + self, index: int, end: int, braces: bool, generics: bool = False + ) -> tuple[int, int | None]: + """Returns the index after the item that runs on from index, and the + index of the brace that opens its body if it has one: the item ends + with the first `;` outside brackets or, if braces, with the first + `{...}` group outside them. In the header of a generic item, the + braces of a const argument inside `<...>`, as in `S<{ 1 }>`, open no + body.""" + tokens = self.tokens + start = index + angles = 0 + while index < end: + token = tokens[index] + if token.kind == PUNCT: + if token.text == ";" and not angles: + return index + 1, None + if token.text == "{" and braces and not angles: + return self.partner[index] + 1, index + if token.text in "([{": + index = self.partner[index] + 1 + continue + if generics and token.text == "<": + angles += 1 + elif ( + generics + and token.text == ">" + and angles + and not is_token(tokens, index - 1, PUNCT, "-", "=") + ): + angles -= 1 + index += 1 + if angles: + # A `<` that no `>` closes is no generic argument list. + return self._scan(start, end, braces) + return end, None + + +@dataclass +class RustFile: + """A parsed Rust file: its tokens, the partners of its brackets, its inner + attributes and items, and every attribute and cfg! invocation in it, with + the index of the invocation's `cfg` and its predicate.""" + + tokens: list[Token] + partner: list[int] + inner_attributes: tuple[Attribute, ...] + items: list[Item] + attributes: list[Attribute] + cfg_macros: list[tuple[int, Cfg]] + + +def parse_rust(text: str) -> RustFile: + """Parses Rust source; raises RustSyntaxError when it does not tokenize or + its brackets do not balance.""" + tokens = tokenize_rust(text) + partner = match_brackets(tokens) + inner, items = _ItemParser(tokens, partner).block(0, len(tokens)) + attributes: list[Attribute] = [] + cfg_macros: list[tuple[int, Cfg]] = [] + for index, token in enumerate(tokens): + if token.kind == PUNCT and token.text == "#": + attribute = attribute_at(tokens, partner, index) + if attribute is not None: + attributes.append(attribute) + elif ( + token.kind == IDENT + and token.text == "cfg" + and is_token(tokens, index + 1, PUNCT, "!") + and is_token(tokens, index + 2, PUNCT, "(", "[", "{") + ): + group = index + 2 + cfg_macros.append( + (index, parse_cfg(tokens, partner, group + 1, partner[group])) + ) + return RustFile(tokens, partner, inner, items, attributes, cfg_macros) + + +# The files of a repository. + + +class Snapshot(Protocol): + """The tracked files of a tree, which may be a work tree or a commit.""" + + @property + def files(self) -> frozenset[str]: ... + + def read(self, path: str) -> bytes: ... + + +def git(root: Path, *arguments: str, stdin: bytes | None = None) -> bytes: + """Runs git in root and returns what it printed; raises CheckError when it + fails.""" + try: + result = subprocess.run( + ["git", *arguments], + cwd=root, + input=stdin, + capture_output=True, + check=False, + ) + except OSError as error: + raise CheckError(f"cannot run git: {error}") from error + if result.returncode != 0: + detail = result.stderr.decode("utf-8", "replace").strip() + raise CheckError(f"git {arguments[0]} failed in {root}: {detail}") + return result.stdout + + +class WorkTree: + """The tracked files of a Git work tree, outside its submodules.""" + + def __init__(self, root: Path) -> None: + self.root = root + files: set[str] = set() + for entry in git(root, "ls-files", "--stage", "-z").split(b"\0"): + if not entry: + continue + metadata, _, name = entry.partition(b"\t") + path = os.fsdecode(name) + if metadata.split()[0] != b"160000" and os.path.lexists(root / path): + files.add(path) + self._files = frozenset(files) + + @property + def files(self) -> frozenset[str]: + return self._files + + def read(self, path: str) -> bytes: + try: + return (self.root / path).read_bytes() + except OSError as error: + raise CheckError(f"cannot read {path}: {error}") from error + + +def decode(data: bytes) -> str: + """Decodes a source file's bytes, with its newlines normalized.""" + return data.decode("utf-8-sig", "replace").replace("\r\n", "\n").replace("\r", "\n") + + +def rust_crates(files: Iterable[str]) -> tuple[str, ...]: + """Returns the directories of the tracked Cargo.toml files, the crates.""" + return tuple( + sorted( + { + posixpath.dirname(path) + for path in files + if posixpath.basename(path) == "Cargo.toml" + } + ) + ) + + +_TOML_TABLE = re.compile(r"^\s*(\[\[?)\s*([^\]]*?)\s*\]\]?\s*(?:#.*)?$") +_TOML_KEY = re.compile(r"""^\s*([A-Za-z0-9_.-]+|"[^"]*"|'[^']*')\s*=\s*(.*?)\s*$""") +_TOML_VALUE = re.compile( + r"""(?:"((?:[^"\\]|\\.)*)"|'([^']*)'|(true|false))\s*(?:#.*)?""" +) +# The directories where Cargo discovers each kind of target but the library, +# and the package settings that turn its discovery of each kind off. +_TARGET_DIRECTORIES = { + "bin": "src/bin", + "test": "tests", + "bench": "benches", + "example": "examples", +} +_DISCOVERY = { + "lib": "autolib", + "bin": "autobins", + "test": "autotests", + "bench": "autobenches", + "example": "autoexamples", +} +# The keys of the package and target tables that target_roots reads. +_TARGET_KEYS = frozenset({"name", "path", "test", *_DISCOVERY.values()}) + + +def _unquoted(key: str) -> str: + """Returns a TOML key or table name of one part without its quotes.""" + if len(key) >= 2 and key[0] == key[-1] and key[0] in "\"'": + return key[1:-1] + return key + + +def _toml_value(text: str) -> str | bool | None: + """Returns the string or boolean that a TOML value on one line spells, or + None for any other value.""" + found = _TOML_VALUE.fullmatch(text) + if found is None: + return None + basic, literal, boolean = found.groups() + if literal is not None: + return literal + if boolean is not None: + return boolean == "true" + try: + # A basic string's escapes are JSON's, but for \U and \e. + return cast(str, json.loads(f'"{basic}"')) + except json.JSONDecodeError: + return None + + +def target_roots( + manifest: str, files: Collection[str], unread: list[tuple[int, str]] | None = None +) -> dict[str, bool]: + """Returns the root files, relative to the crate, of the targets of a + Cargo manifest, each with whether `cargo test` runs its tests: the + library, the binaries, the integration tests, the benchmarks, and the + examples that its tables name, at their `path` or where Cargo infers it, + and that Cargo discovers among files, the crate's files relative to it, + unless the package turns that off. `cargo test` runs the tests of the + library, the binaries, and the integration tests whose `test` key is not + false, and of the benchmarks and examples whose `test` key is true. Reads + the keys of the package and target tables that Cargo.toml files spell one + per line, with a string or a boolean, and appends to unread the line and + the problem of each declaration of a target that it cannot read, such as + an inline table, which it ignores.""" + package: dict[str, str | bool] = {} + tables: list[tuple[str, dict[str, str | bool]]] = [] + current: dict[str, str | bool] | None = None + section: str | None = None + for number, line in enumerate(manifest.splitlines(), 1): + table = _TOML_TABLE.match(line) + if table: + array, section = table.group(1) == "[[", _unquoted(table.group(2)) + current = None + if not array and section == "package": + current = package + elif (array and section in _TARGET_DIRECTORIES) or ( + not array and section == "lib" + ): + current = {} + tables.append((section, current)) + continue + assignment = _TOML_KEY.match(line) + if assignment is None: + continue + key, value = _unquoted(assignment.group(1)), assignment.group(2) + if section is None and key.split(".")[0].strip("\"'") in ( + "package", + "lib", + *_TARGET_DIRECTORIES, + ): + if unread is not None: + unread.append((number, f"`{key}` declares targets outside a table")) + continue + if current is None or key not in _TARGET_KEYS: + continue + read = _toml_value(value) + if read is None: + if unread is not None: + unread.append( + (number, f"`{key}` holds no string or boolean on its line") + ) + continue + current[key] = read + name = package.get("name") + package_name = name if isinstance(name, str) else "" + # Each target, by its kind and name: its root, and whether `cargo test` + # runs its tests. A table that names a target that Cargo discovers + # configures that target. + targets: dict[tuple[str, str], tuple[str, bool]] = {} + if package.get(_DISCOVERY["lib"]) is not False and "src/lib.rs" in files: + targets["lib", ""] = ("src/lib.rs", True) + for kind, directory in _TARGET_DIRECTORIES.items(): + if package.get(_DISCOVERY[kind]) is False: + continue + tested = kind in ("bin", "test") + for path in sorted(files): + parent, file = posixpath.split(path) + if parent == directory and file.endswith(".rs"): + targets[kind, file[: -len(".rs")]] = (path, tested) + elif posixpath.dirname(parent) == directory and file == "main.rs": + targets.setdefault((kind, posixpath.basename(parent)), (path, tested)) + if kind == "bin" and package_name and "src/main.rs" in files: + targets["bin", package_name] = ("src/main.rs", True) + for kind, table in tables: + target = table.get("name") + target_name = target if isinstance(target, str) else "" + path = table.get("path") + if isinstance(path, str): + root = posixpath.normpath(path) + elif kind == "lib": + root = "src/lib.rs" + elif target_name: + directory = _TARGET_DIRECTORIES[kind] + inferred = [f"{directory}/{target_name}.rs"] + inferred.append(f"{directory}/{target_name}/main.rs") + if kind == "bin" and target_name == package_name: + inferred.insert(0, "src/main.rs") + root = next((found for found in inferred if found in files), inferred[0]) + else: + continue + test = table.get("test") + tested = test if isinstance(test, bool) else kind in ("lib", "bin", "test") + targets[kind, "" if kind == "lib" else target_name] = (root, tested) + roots: dict[str, bool] = {} + for root, tested in targets.values(): + roots[root] = roots.get(root, False) or tested + return roots + + +def crate_of(path: str, crates: Sequence[str]) -> str | None: + """Returns the innermost crate that holds path.""" + found: str | None = None + for crate in crates: + if (not crate or path.startswith(crate + "/")) and ( + found is None or len(crate) > len(found) + ): + found = crate + return found + + +def crate_relative(path: str, crate: str) -> str: + """Returns a path of a crate relative to the crate's directory.""" + return path[len(crate) + 1 :] if crate else path + + +def rust_role(path: str, crate: str) -> str: + """Returns "production" for a production file of the crate, "test" for a + test file, and "" for any other file.""" + if not path.endswith(".rs"): + return "" + relative = crate_relative(path, crate) + if relative.startswith("tests/"): + return "test" + if not relative.startswith("src/"): + return "" + parts = relative.split("/")[1:] + if parts[-1] == "tests.rs" or "tests" in parts[:-1]: + return "test" + return "production" + + +def _join(*parts: str) -> str: + return "/".join(part for part in parts if part) + + +def is_crate_root(source: str) -> bool: + """Whether a file, relative to its crate's src/, roots a crate: the + library, the main binary, or another binary.""" + directory, name = posixpath.split(source) + return ( + (not directory and name in ("lib.rs", "main.rs")) + or directory == "bin" + or (posixpath.dirname(directory) == "bin" and name == "main.rs") + ) + + +def is_target_root(source: str, base: str = "src") -> bool: + """Whether a file, relative to its crate's base directory, src or tests, + roots one of the crate's targets: in src/, as is_crate_root says, and in + tests/, an integration test, `NAME.rs` or `NAME/main.rs`.""" + if base != "tests": + return is_crate_root(source) + directory, name = posixpath.split(source) + return not directory or ("/" not in directory and name == "main.rs") + + +def module_directory(source: str, base: str = "src", root: bool | None = None) -> str: + """Returns the directory, relative to the crate's base directory, of the + modules that a file of that directory declares: the file's own for a + target's root or a mod.rs file, and the one named after the file for any + other. root says whether the file roots a target, which by default the + places where Cargo discovers targets say.""" + directory, name = posixpath.split(source) + if root is None: + root = is_target_root(source, base) + if name == "mod.rs" or root: + return directory + return _join(directory, name[: -len(".rs")]) + + +def module_files( + crate: str, + source: str, + inline: Sequence[str], + name: str, + base: str = "src", + root: bool | None = None, +) -> tuple[str, str]: + """Returns the two files that may hold the module that `mod NAME;` + declares in a file of the crate's base directory, src or tests, inside the + inline modules inline; root says whether the file roots a target.""" + directory = _join(crate, base, module_directory(source, base, root), *inline) + return _join(directory, f"{name}.rs"), _join(directory, name, "mod.rs") + + +def declared_files( + crate: str, + base: str, + source: str, + declaration: Declaration, + root: bool | None = None, +) -> tuple[str, ...]: + """Returns the files that may hold the module that a declaration of a file + of the crate's base directory declares; root says whether the file roots a + target. An integration test may set a module's file with #[path], + relative to the declaring file's directory or, inside inline modules, to + their directory.""" + if declaration.path is None or base != "tests": + return module_files( + crate, source, declaration.inline, declaration.name, base, root + ) + if declaration.inline: + directory = _join(crate, base, module_directory(source, base, root)) + directory = _join(directory, *declaration.inline) + else: + directory = _join(crate, base, posixpath.dirname(source)) + return (posixpath.normpath(_join(directory, declaration.path)),) + + +def module_path(source: str) -> tuple[str, ...]: + """Returns the module path, from its crate's root, of a file of src/, such + as ("host", "store") for host/store/mod.rs; a binary's root starts with + "bin:NAME".""" + parts = source[: -len(".rs")].split("/") + if parts[0] == "bin": + binary = parts[1] + rest = parts[2:] + if rest == ["main"]: + rest = [] + return (f"bin:{binary}", *rest) + if parts in (["lib"], ["main"]): + return () + if parts[-1] == "mod": + parts = parts[:-1] + return tuple(parts) + + +# The test code of the production files. + + +@dataclass(frozen=True) +class Declaration: + """A `mod NAME;` declaration, inside the inline modules inline, and whether + a cfg that requires `test` guards it; the file that its #[path] sets, if + it has one; and whether a cfg that no build of tests satisfies guards it + or a module that holds it, so that tests never compile its module.""" + + inline: tuple[str, ...] + name: str + line: int + test_only: bool + path: str | None = None + excluded: bool = False + + +@dataclass +class RustAnalysis: + """What a Rust file of a crate's src/ or tests/ declares, and whether its + inner attributes keep tests from compiling it; and, for a production file, + the test code that it holds: the lines of its test hooks, the number of + its tests, and the test code that only the baseline allows.""" + + path: str + crate: str + production: bool + declarations: list[Declaration] = field(default_factory=list[Declaration]) + hooks: list[int] = field(default_factory=list[int]) + tests: int = 0 + legacy: list[Problem] = field(default_factory=list[Problem]) + problems: list[Problem] = field(default_factory=list[Problem]) + excluded: bool = False + + +def _describe(item: Item) -> str: + if item.kind == "impl": + return f"`{item.name}` block" + if item.kind in ("macro", "macro_rules"): + return f"`{item.name}!`" if item.kind == "macro" else f"macro `{item.name}`" + if item.kind == "use": + return "`use`" + if item.name: + return f"`{item.kind} {item.name}`" + return f"`{item.kind}` item" + + +def _path_value(tokens: Sequence[Token], attributes: Iterable[Attribute]) -> str | None: + """Returns the file that a `#[path = "FILE"]` among attributes sets.""" + for attribute in attributes: + if attribute.path == "path": + for token in tokens[attribute.start : attribute.end]: + if token.kind == LITERAL and token.text.startswith('"'): + return token.text[1:-1] + return None + + +def analyze_rust(path: str, crate: str, text: str) -> RustAnalysis: + """Finds the module declarations of a Rust file of a crate's src/ or + tests/ and, if it is a production file, its test code.""" + production = rust_role(path, crate) == "production" + analysis = RustAnalysis(path, crate, production) + try: + parsed = parse_rust(text) + except RustSyntaxError as error: + analysis.problems.append( + Problem(path, error.line, "rust-syntax", f"cannot read the file: {error}") + ) + return analysis + tokens, partner = parsed.tokens, parsed.partner + base, _, source = crate_relative(path, crate).partition("/") + + def tests_file(inline: Sequence[str]) -> str: + return module_files(crate, source, inline, "tests", base)[0] + + def excludes(attributes: Iterable[Attribute]) -> bool: + return any( + attribute_excludes_tests(tokens, partner, attribute) + for attribute in attributes + ) + + exempt: set[int] = set() + legacy_ranges: list[tuple[int, int]] = [] + + def legacy(line: int, rule: str, message: str) -> None: + analysis.legacy.append(Problem(path, line, rule, message)) + + def walk( + items: Sequence[Item], inline: tuple[str, ...], in_tests: bool, excluded: bool + ) -> None: + """Walks items outside function bodies. In test code, which a test file + or a module that only tests compile holds, it only collects the module + declarations. excluded says whether the inline modules that hold items + compile in no build of tests.""" + for item in items: + only_tests = any( + attribute_requires_test(tokens, partner, attribute) + for attribute in item.attributes + ) + gone = excluded or excludes(item.attributes) + for attribute in item.attributes: + if base == "src" and "path" in attribute_paths( + tokens, partner, attribute + ): + analysis.problems.append( + Problem( + path, + attribute.line, + "path-attribute", + "#[path] sets the file of a module, which the test " + "layout does not follow; name the file after its " + "module", + ) + ) + if item.kind == "mod" and item.body is None: + analysis.declarations.append( + Declaration( + inline, + item.name, + item.line, + only_tests or in_tests, + _path_value(tokens, item.attributes), + gone, + ) + ) + if in_tests: + continue + if only_tests and item.name == "tests": + exempt.update(attribute.start for attribute in item.attributes) + elif only_tests: + legacy( + item.line, + "test-module-name", + f"module `{item.name}` compiles only for tests, but only " + "the `tests` module may; move it into the `tests` module, " + "as " + f"{module_files(crate, source, (*inline, 'tests'), item.name, base)[0]}", + ) + elif item.name == "tests": + analysis.problems.append( + Problem( + path, + item.line, + "tests-module", + "module `tests` compiles outside tests; guard it with " + "#[cfg(test)]", + ) + ) + continue + inner = gone or excludes(item.inner_attributes) + if in_tests: + if item.kind == "mod": + walk(item.children, (*inline, item.name), True, inner) + continue + if item.kind == "mod": + only_tests = only_tests or any( + attribute_requires_test(tokens, partner, attribute) + for attribute in item.inner_attributes + ) + if only_tests: + walk(item.children, (*inline, item.name), True, inner) + legacy_ranges.append((item.start, item.end)) + if item.name == "tests": + message = ( + f"inline module `tests`; move its contents to " + f"{tests_file(inline)} and declare it as " + "`#[cfg(test)] mod tests;`" + ) + else: + message = ( + f"inline module `{item.name}` that compiles only for " + "tests; move it into the `tests` module, as " + f"{module_files(crate, source, (*inline, 'tests'), item.name, base)[0]}" + ) + legacy(item.line, "inline-test-module", message) + continue + if item.name == "tests": + analysis.problems.append( + Problem( + path, + item.line, + "tests-module", + "inline module `tests` compiles outside tests; guard " + f"it with #[cfg(test)] and move it to {tests_file(inline)}", + ) + ) + walk(item.children, (*inline, item.name), False, inner) + continue + if only_tests: + legacy_ranges.append((item.start, item.end)) + legacy( + item.line, + "test-only-item", + f"{_describe(item)} compiles only for tests; move it into the " + f"`tests` module, {tests_file(inline)}", + ) + continue + if item.kind == "macro" and item.name.split("::")[-1] == "proptest": + legacy_ranges.append((item.start, item.end)) + legacy( + item.line, + "test-harness", + "proptest! block; tests belong in the `tests` module, " + f"{tests_file(inline)}", + ) + continue + walk(item.children, inline, False, gone) + + analysis.excluded = excludes(parsed.inner_attributes) + walk(parsed.items, (), not production, False) + for attribute in parsed.inner_attributes: + if base == "src" and "path" in attribute_paths(tokens, partner, attribute): + analysis.problems.append( + Problem( + path, + attribute.line, + "path-attribute", + "#![path] sets the file of a module, which the test layout " + "does not follow", + ) + ) + if production and attribute_requires_test(tokens, partner, attribute): + legacy( + attribute.line, + "test-only-file", + "#![cfg(test)] makes this production file compile only for tests; " + "move its code into a `tests` module", + ) + if not production: + return analysis + + def in_legacy(index: int) -> bool: + return any(start <= index < end for start, end in legacy_ranges) + + for attribute in parsed.attributes: + if attribute.start not in exempt and attribute_mentions_test( + tokens, partner, attribute + ): + analysis.hooks.append(attribute.line) + # cfg_attr may apply a test attribute, as #[cfg_attr(test, test)] does. + paths = attribute_paths(tokens, partner, attribute) + if any(is_test_function_attribute(path) for path in paths): + analysis.tests += 1 + harness = [path for path in paths if is_test_attribute(path)] + if harness and not in_legacy(attribute.start): + legacy( + attribute.line, + "test-harness", + f"#[{harness[0]}] in a production file; tests belong in the " + f"`tests` module", + ) + for index, cfg in parsed.cfg_macros: + if mentions_test(cfg): + analysis.hooks.append(tokens[index].line) + analysis.hooks.sort() + return analysis + + +# The allowlist. + + +@dataclass +class Allowlist: + """The test hooks and tests that production files may hold: the counts of + the baseline files, which predate the layout, and the test hooks of the + seams.""" + + baseline: dict[str, tuple[int, int]] = field( + default_factory=dict[str, tuple[int, int]] + ) + seams: dict[str, int] = field(default_factory=dict[str, int]) + + +def parse_allowlist(text: str) -> tuple[Allowlist, list[str]]: + """Parses the allowlist; returns it and what is wrong with its form.""" + errors: list[str] = [] + allowlist = Allowlist() + try: + data: object = json.loads(text) + except json.JSONDecodeError as error: + return allowlist, [f"is not JSON: {error}"] + if not isinstance(data, dict): + return allowlist, ["is not a JSON object"] + document = cast(dict[str, object], data) + expected = {"schema", "schema_version", "baseline", "seams"} + unknown = sorted(set(document) - expected) + if unknown: + errors.append(f"has unknown keys {', '.join(unknown)}") + if document.get("schema") != ALLOWLIST_SCHEMA: + errors.append(f'schema is not "{ALLOWLIST_SCHEMA}"') + if document.get("schema_version") != ALLOWLIST_SCHEMA_VERSION: + errors.append(f"schema_version is not {ALLOWLIST_SCHEMA_VERSION}") + baseline = document.get("baseline", {}) + if not isinstance(baseline, dict): + errors.append("baseline is not an object") + else: + for path, counts in cast(dict[str, object], baseline).items(): + if ( + isinstance(counts, dict) + and set(cast(dict[str, object], counts)) == {"hooks", "tests"} + and all( + _is_count(value) + for value in cast(dict[str, object], counts).values() + ) + ): + entry = cast(dict[str, int], counts) + allowlist.baseline[path] = (entry["hooks"], entry["tests"]) + else: + errors.append( + f'baseline entry {path} is not {{"hooks": COUNT, "tests": COUNT}}' + ) + seams = document.get("seams", {}) + if not isinstance(seams, dict): + errors.append("seams is not an object") + else: + for path, count in cast(dict[str, object], seams).items(): + if _is_count(count) and cast(int, count) > 0: + allowlist.seams[path] = cast(int, count) + else: + errors.append(f"seams entry {path} is not a positive count") + if not errors and text != render_allowlist(allowlist): + errors.append( + "is not in its canonical form, with sorted entries, two-space " + f"indentation, and a final newline; run `{TIGHTEN_COMMAND}`" + ) + return allowlist, errors + + +def _is_count(value: object) -> bool: + return isinstance(value, int) and not isinstance(value, bool) and value >= 0 + + +def render_allowlist(allowlist: Allowlist) -> str: + """Returns the canonical text of the allowlist.""" + document = { + "schema": ALLOWLIST_SCHEMA, + "schema_version": ALLOWLIST_SCHEMA_VERSION, + "baseline": { + path: {"hooks": hooks, "tests": tests} + for path, (hooks, tests) in sorted(allowlist.baseline.items()) + }, + "seams": dict(sorted(allowlist.seams.items())), + } + return json.dumps(document, indent=2) + "\n" + + +def tighten(allowlist: Allowlist, analyses: Mapping[str, RustAnalysis]) -> Allowlist: + """Lowers the allowlist to what the production files hold. + + A baseline file whose test code but its hooks moved out moves to the + seams. An entry for a file that is no production file goes. No entry + rises, and no file that the allowlist does not list joins it. + """ + tightened = Allowlist() + for path, (hooks, tests) in allowlist.baseline.items(): + analysis = analyses.get(path) + if analysis is None or not analysis.production: + continue + found = (len(analysis.hooks), analysis.tests) + if found[0] > hooks or found[1] > tests: + tightened.baseline[path] = (hooks, tests) + elif analysis.legacy or analysis.tests: + tightened.baseline[path] = found + elif found[0]: + tightened.seams[path] = found[0] + for path, count in allowlist.seams.items(): + analysis = analyses.get(path) + if analysis is None or not analysis.production: + continue + if path in tightened.baseline or path in tightened.seams: + continue + found = len(analysis.hooks) + if found > count: + tightened.seams[path] = count + elif found: + tightened.seams[path] = found + return tightened + + +# The checks. + + +def _lines(lines: Sequence[int]) -> str: + shown = ", ".join(str(line) for line in lines[:12]) + return shown + (", ..." if len(lines) > 12 else "") + + +def _rust_count_problems(analysis: RustAnalysis, allowlist: Allowlist) -> list[Problem]: + path = analysis.path + hooks = len(analysis.hooks) + if path in allowlist.baseline: + allowed_hooks, allowed_tests = allowlist.baseline[path] + if not analysis.legacy and not analysis.tests: + if hooks > allowed_hooks: + return [ + Problem( + path, + 0, + "test-hooks", + f"holds {hooks} test hook(s), at lines " + f"{_lines(analysis.hooks)}, but the baseline allows " + f"{allowed_hooks}: move test code into the `tests` module", + ) + ] + return [ + Problem( + path, + 0, + "allowlist", + "the baseline lists this file, whose tests moved out; run " + f"`{TIGHTEN_COMMAND}` to move it to the seams", + ) + ] + problems: list[Problem] = [] + if hooks > allowed_hooks or analysis.tests > allowed_tests: + problems.append( + Problem( + path, + 0, + "test-code", + f"holds {hooks} test hook(s) and {analysis.tests} test(s), " + f"but the baseline allows {allowed_hooks} and {allowed_tests}: " + "this file predates the test layout, so new tests and test " + "code go in its `tests` module; move its tests there first", + ) + ) + elif hooks < allowed_hooks or analysis.tests < allowed_tests: + problems.append( + Problem( + path, + 0, + "allowlist", + f"holds {hooks} test hook(s) and {analysis.tests} test(s), " + f"fewer than the {allowed_hooks} and {allowed_tests} that the " + f"baseline allows; run `{TIGHTEN_COMMAND}`", + ) + ) + return problems + problems = list(analysis.legacy) + allowed = allowlist.seams.get(path, 0) + if hooks > allowed: + problems.append( + Problem( + path, + 0, + "test-hooks", + f"holds {hooks} test hook(s), at lines {_lines(analysis.hooks)}, but " + f"the allowlist allows {allowed}: move test code into the `tests` " + "module, and allow only a hook that cannot move in the seams of " + f"{ALLOWLIST}", + ) + ) + elif hooks < allowed: + problems.append( + Problem( + path, + 0, + "allowlist", + f"holds {hooks} test hook(s), fewer than the {allowed} that the " + f"seams allow; run `{TIGHTEN_COMMAND}`", + ) + ) + return problems + + +def analyze_crates(snapshot: Snapshot) -> tuple[dict[str, RustAnalysis], list[Problem]]: + """Analyzes every Rust file of the crates' src/ directories and of their + integration tests but the data of TEST_DATA_DIRECTORIES; returns the + analyses and the problems of the module tree: `tests` modules whose files + are missing, and test files that no target of their crate whose tests + `cargo test` runs, as target_roots finds them, compiles, since no module + that such a target's root reaches declares them, through declarations + that builds of tests compile.""" + files = snapshot.files + crates = rust_crates(files) + members: dict[str, set[str]] = {crate: set() for crate in crates} + for path in files: + crate = crate_of(path, crates) + if crate is not None: + members[crate].add(crate_relative(path, crate)) + targets: dict[str, dict[str, bool]] = {} + problems: list[Problem] = [] + for crate in crates: + manifest = _join(crate, "Cargo.toml") + unread: list[tuple[int, str]] = [] + targets[crate] = target_roots( + decode(snapshot.read(manifest)), members[crate], unread + ) + problems.extend( + Problem(manifest, line, "manifest", f"cannot read its targets: {message}") + for line, message in unread + ) + analyses: dict[str, RustAnalysis] = {} + for path in sorted(files): + crate = crate_of(path, crates) + if crate is None or not path.endswith(".rs"): + continue + relative = crate_relative(path, crate) + if relative.startswith("src/") or ( + relative.startswith("tests/") + and not relative.startswith(TEST_DATA_DIRECTORIES) + ): + analyses[path] = analyze_rust(path, crate, decode(snapshot.read(path))) + declares: dict[str, list[str]] = {} + for path, analysis in analyses.items(): + relative = crate_relative(path, analysis.crate) + base, _, source = relative.partition("/") + # A target's root declares the modules of its directory only where + # its own target compiles it, which for tests is where it runs them. + root = targets[analysis.crate].get(relative, False) + for declaration in analysis.declarations: + candidates = declared_files(analysis.crate, base, source, declaration, root) + present = [candidate for candidate in candidates if candidate in files] + if not declaration.excluded: + declares.setdefault(path, []).extend(present) + if ( + analysis.production + and declaration.test_only + and declaration.name == "tests" + and len(present) != 1 + ): + problems.append( + Problem( + path, + declaration.line, + "tests-module", + f"`mod tests;` names {' and '.join(candidates)}, of which " + "exactly one must be a tracked file" + if present + else f"`mod tests;` names neither {candidates[0]} nor " + f"{candidates[1]}; stage the file of the module", + ) + ) + compiled = { + path + for path, analysis in analyses.items() + if targets[analysis.crate].get(crate_relative(path, analysis.crate)) + and not analysis.excluded + } + pending = sorted(compiled) + while pending: + for declared in declares.get(pending.pop(), ()): + target = analyses.get(declared) + if declared not in compiled and not (target and target.excluded): + compiled.add(declared) + pending.append(declared) + for path, analysis in analyses.items(): + if analysis.production or path in compiled: + continue + problems.append( + Problem( + path, + 0, + "orphan-test-file", + "no target of the crate whose tests run compiles this test file, " + "so its tests never run; declare it with `mod NAME;` in the file " + "of its parent module, which a target of its crate whose tests " + "`cargo test` runs, its library, a binary, or an integration test, " + "must reach, through declarations that builds of tests compile", + ) + ) + return analyses, problems + + +def python_problems( + path: str, text: str, files: frozenset[str] = frozenset() +) -> list[Problem]: + """Checks where a tracked Python module lives and what it imports. files, + the tracked files, name the test modules that an import may resolve to.""" + name = posixpath.basename(path) + if name.startswith("test_"): + if PYTHON_UNIT_TESTS.fullmatch(path) is None: + return [ + Problem( + path, + 0, + "python-test-location", + "Python unit tests live in scripts/, .github/specula/tests/, " + "or .github/skills/*/scripts/", + ) + ] + return [] + problems: list[Problem] = [] + if name.endswith("_tests.py") and PYTHON_END_TO_END_TESTS.fullmatch(path) is None: + problems.append( + Problem( + path, + 0, + "python-test-location", + "the end-to-end scenario drivers, the `*_tests.py` modules, live " + "in scripts/nvx_tools/", + ) + ) + if path in PYTHON_TEST_SUPPORT: + return problems + try: + tree = ast.parse(text, path) + except (SyntaxError, ValueError, RecursionError, MemoryError) as error: + line = getattr(error, "lineno", None) + problems.append( + Problem( + path, + line if isinstance(line, int) else 0, + "python-syntax", + f"cannot read the module: {error}", + ) + ) + return problems + support = { + posixpath.basename(module)[: -len(".py")] for module in PYTHON_TEST_SUPPORT + } + tests = PYTHON_TEST_SUPPORT | { + file + for file in files + if file.endswith(".py") and posixpath.basename(file).startswith("test_") + } + + def test_module(parts: Sequence[str], directory: str | None) -> bool: + """Whether a module's path, from directory or, if None, from any import + root, names a tracked test module.""" + suffix = "/".join(parts) + ".py" + if directory is not None: + return _join(directory, suffix) in tests + return any(test == suffix or test.endswith("/" + suffix) for test in tests) + + for node in ast.walk(tree): + found: list[str] = [] + line = 0 + if isinstance(node, ast.Import): + line = node.lineno + for alias in node.names: + parts = alias.name.split(".") + if ( + parts[0] in PYTHON_TEST_FRAMEWORKS + or any(part.startswith("test_") for part in parts) + or parts[-1] in support + ): + found.append(alias.name) + elif isinstance(node, ast.ImportFrom): + line = node.lineno + parts = node.module.split(".") if node.module else [] + directory: str | None = None + if node.level: + directory = posixpath.dirname(path) + for _ in range(node.level - 1): + directory = posixpath.dirname(directory) + source = "." * node.level + (node.module or "") + if parts and ( + (not node.level and parts[0] in PYTHON_TEST_FRAMEWORKS) + or any(part.startswith("test_") for part in parts) + or parts[-1] in support + or test_module(parts, directory) + ): + found.append(source) + # A name may be a module of the package, such as `from scripts + # import test_a`, or an item of the module, such as a function + # named test_a, which no test module's path matches. + found.extend( + f"{source}.{alias.name}" + for alias in node.names + if alias.name != "*" + and ( + alias.name in support + or test_module([*parts, alias.name], directory) + ) + ) + for module in found: + problems.append( + Problem( + path, + line, + "python-test-import", + f"imports {module}, which only tests may; move the code " + "that needs it into a test module", + ) + ) + return problems + + +@dataclass +class Layout: + """The checked layout: the analyses of the Rust files, the allowlist, and + the problems.""" + + analyses: dict[str, RustAnalysis] + allowlist: Allowlist + problems: list[Problem] + + +def check_snapshot(snapshot: Snapshot) -> Layout: + """Checks the test layout of a tree.""" + analyses, problems = analyze_crates(snapshot) + allowlist = Allowlist() + if ALLOWLIST in snapshot.files: + allowlist, errors = parse_allowlist(decode(snapshot.read(ALLOWLIST))) + problems.extend(Problem(ALLOWLIST, 0, "allowlist", error) for error in errors) + for path in sorted(set(allowlist.baseline) | set(allowlist.seams)): + analysis = analyses.get(path) + if analysis is None or not analysis.production: + problems.append( + Problem( + ALLOWLIST, + 0, + "allowlist", + f"lists {path}, which is no tracked production file of a " + f"crate's src/; run `{TIGHTEN_COMMAND}`", + ) + ) + elif path in allowlist.baseline and path in allowlist.seams: + problems.append( + Problem( + ALLOWLIST, + 0, + "allowlist", + f"lists {path} in both the baseline and the seams", + ) + ) + for analysis in analyses.values(): + problems.extend(analysis.problems) + if analysis.production: + problems.extend(_rust_count_problems(analysis, allowlist)) + for path in sorted(snapshot.files): + if path.endswith(".py"): + problems.extend( + python_problems(path, decode(snapshot.read(path)), snapshot.files) + ) + return Layout(analyses, allowlist, sorted(set(problems))) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Check that tests live apart from the code that they test." + ) + parser.add_argument( + "--root", + type=Path, + default=REPO_ROOT, + help="Git work tree to check (default: this repository)", + ) + parser.add_argument( + "--tighten", + action="store_true", + help=f"lower {ALLOWLIST} to what the production files hold, then check", + ) + args = parser.parse_args(argv) + root: Path = args.root + try: + tree = WorkTree(root) + layout = check_snapshot(tree) + if args.tighten: + # A file that does not parse counts no test code, and an allowlist + # that does not parse lacks its broken entries, so tightening then + # would drop what they allow. + unread = [ + str(problem) + for problem in layout.problems + if problem.rule == "rust-syntax" + ] + if ALLOWLIST in tree.files: + errors = parse_allowlist(decode(tree.read(ALLOWLIST)))[1] + unread.extend(f"{ALLOWLIST}: {error}" for error in errors) + if unread: + for line in unread: + print(line) + print( + f"error: {ALLOWLIST} is not tightened while the files above " + "do not parse", + file=sys.stderr, + ) + return 1 + text = render_allowlist(tighten(layout.allowlist, layout.analyses)) + (root / ALLOWLIST).write_text(text, encoding="utf-8", newline="\n") + layout = check_snapshot(WorkTree(root)) + except CheckError as error: + print(f"error: {error}", file=sys.stderr) + return 2 + annotate = os.environ.get("GITHUB_ACTIONS") == "true" + for problem in layout.problems: + print(problem.annotation() if annotate else problem) + if layout.problems: + print( + f"error: {len(layout.problems)} problem(s) with the test layout; " + "see doc/contribute.md#test-layout", + file=sys.stderr, + ) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test-layout-allowlist.json b/scripts/test-layout-allowlist.json new file mode 100644 index 000000000..d9c6d2e50 --- /dev/null +++ b/scripts/test-layout-allowlist.json @@ -0,0 +1,603 @@ +{ + "schema": "nvx-test-layout-allowlist", + "schema_version": 1, + "baseline": { + "nvx-rs/src/contract.rs": { + "hooks": 1, + "tests": 2 + }, + "nvx-rs/src/error.rs": { + "hooks": 1, + "tests": 5 + }, + "nvx-rs/src/gpt/crc.rs": { + "hooks": 1, + "tests": 2 + }, + "nvx-rs/src/guest/cgroup.rs": { + "hooks": 5, + "tests": 17 + }, + "nvx-rs/src/guest/clone/mounts.rs": { + "hooks": 1, + "tests": 3 + }, + "nvx-rs/src/guest/console_pending.rs": { + "hooks": 1, + "tests": 5 + }, + "nvx-rs/src/guest/device_io.rs": { + "hooks": 1, + "tests": 10 + }, + "nvx-rs/src/guest/entry.rs": { + "hooks": 2, + "tests": 0 + }, + "nvx-rs/src/guest/exec.rs": { + "hooks": 5, + "tests": 0 + }, + "nvx-rs/src/guest/exit.rs": { + "hooks": 1, + "tests": 3 + }, + "nvx-rs/src/guest/generation.rs": { + "hooks": 1, + "tests": 1 + }, + "nvx-rs/src/guest/gpt.rs": { + "hooks": 1, + "tests": 2 + }, + "nvx-rs/src/guest/ingress/address.rs": { + "hooks": 1, + "tests": 3 + }, + "nvx-rs/src/guest/ingress/announcer.rs": { + "hooks": 1, + "tests": 5 + }, + "nvx-rs/src/guest/ingress/collect.rs": { + "hooks": 1, + "tests": 10 + }, + "nvx-rs/src/guest/ingress/diag.rs": { + "hooks": 1, + "tests": 2 + }, + "nvx-rs/src/guest/ingress/encode.rs": { + "hooks": 1, + "tests": 3 + }, + "nvx-rs/src/guest/ingress/mod.rs": { + "hooks": 1, + "tests": 1 + }, + "nvx-rs/src/guest/ingress/sender.rs": { + "hooks": 1, + "tests": 7 + }, + "nvx-rs/src/guest/ingress/table.rs": { + "hooks": 1, + "tests": 11 + }, + "nvx-rs/src/guest/ingress/test_mode.rs": { + "hooks": 1, + "tests": 1 + }, + "nvx-rs/src/guest/kernel_text.rs": { + "hooks": 1, + "tests": 5 + }, + "nvx-rs/src/guest/managed_agent/cgroup.rs": { + "hooks": 1, + "tests": 6 + }, + "nvx-rs/src/guest/managed_agent/dispatch.rs": { + "hooks": 1, + "tests": 32 + }, + "nvx-rs/src/guest/managed_agent/environment.rs": { + "hooks": 1, + "tests": 2 + }, + "nvx-rs/src/guest/managed_agent/exec.rs": { + "hooks": 1, + "tests": 1 + }, + "nvx-rs/src/guest/managed_agent/exec_config.rs": { + "hooks": 2, + "tests": 3 + }, + "nvx-rs/src/guest/managed_agent/helper.rs": { + "hooks": 1, + "tests": 4 + }, + "nvx-rs/src/guest/managed_agent/launch.rs": { + "hooks": 1, + "tests": 8 + }, + "nvx-rs/src/guest/managed_agent/loopback.rs": { + "hooks": 1, + "tests": 2 + }, + "nvx-rs/src/guest/managed_agent/mappings.rs": { + "hooks": 3, + "tests": 13 + }, + "nvx-rs/src/guest/managed_agent/mod.rs": { + "hooks": 5, + "tests": 3 + }, + "nvx-rs/src/guest/managed_agent/seam.rs": { + "hooks": 8, + "tests": 0 + }, + "nvx-rs/src/guest/mappings.rs": { + "hooks": 3, + "tests": 0 + }, + "nvx-rs/src/guest/mappings/privileged.rs": { + "hooks": 0, + "tests": 10 + }, + "nvx-rs/src/guest/mappings/table.rs": { + "hooks": 1, + "tests": 10 + }, + "nvx-rs/src/guest/mmio_write.rs": { + "hooks": 1, + "tests": 4 + }, + "nvx-rs/src/guest/parse.rs": { + "hooks": 1, + "tests": 6 + }, + "nvx-rs/src/guest/process.rs": { + "hooks": 1, + "tests": 9 + }, + "nvx-rs/src/guest/reseed.rs": { + "hooks": 2, + "tests": 10 + }, + "nvx-rs/src/guest/root.rs": { + "hooks": 1, + "tests": 0 + }, + "nvx-rs/src/guest/root/block.rs": { + "hooks": 1, + "tests": 4 + }, + "nvx-rs/src/guest/root/bootstrap.rs": { + "hooks": 1, + "tests": 2 + }, + "nvx-rs/src/guest/root/kernel.rs": { + "hooks": 2, + "tests": 0 + }, + "nvx-rs/src/guest/root/share.rs": { + "hooks": 1, + "tests": 5 + }, + "nvx-rs/src/guest/spawn.rs": { + "hooks": 5, + "tests": 22 + }, + "nvx-rs/src/guest/status.rs": { + "hooks": 2, + "tests": 1 + }, + "nvx-rs/src/guest/sys/clock.rs": { + "hooks": 1, + "tests": 5 + }, + "nvx-rs/src/guest/sys/discipline.rs": { + "hooks": 1, + "tests": 3 + }, + "nvx-rs/src/guest/sys/entry.rs": { + "hooks": 5, + "tests": 2 + }, + "nvx-rs/src/guest/sys/fd.rs": { + "hooks": 3, + "tests": 0 + }, + "nvx-rs/src/guest/sys/ioctl.rs": { + "hooks": 1, + "tests": 7 + }, + "nvx-rs/src/guest/sys/kernel.rs": { + "hooks": 1, + "tests": 2 + }, + "nvx-rs/src/guest/sys/mod.rs": { + "hooks": 12, + "tests": 10 + }, + "nvx-rs/src/guest/sys/msr.rs": { + "hooks": 1, + "tests": 1 + }, + "nvx-rs/src/guest/sys/poll.rs": { + "hooks": 1, + "tests": 1 + }, + "nvx-rs/src/guest/sys/port.rs": { + "hooks": 3, + "tests": 9 + }, + "nvx-rs/src/guest/sys/process.rs": { + "hooks": 11, + "tests": 6 + }, + "nvx-rs/src/guest/sys/rooted.rs": { + "hooks": 6, + "tests": 7 + }, + "nvx-rs/src/guest/sys/scheduler.rs": { + "hooks": 1, + "tests": 2 + }, + "nvx-rs/src/guest/sys/secret.rs": { + "hooks": 1, + "tests": 1 + }, + "nvx-rs/src/guest/sys/signal.rs": { + "hooks": 4, + "tests": 5 + }, + "nvx-rs/src/guest/sys/sock_diag.rs": { + "hooks": 5, + "tests": 5 + }, + "nvx-rs/src/guest/sys/socket.rs": { + "hooks": 1, + "tests": 3 + }, + "nvx-rs/src/guest/sys/spawn.rs": { + "hooks": 23, + "tests": 15 + }, + "nvx-rs/src/guest/sys/timer.rs": { + "hooks": 1, + "tests": 1 + }, + "nvx-rs/src/guest/sys/x86.rs": { + "hooks": 1, + "tests": 3 + }, + "nvx-rs/src/guest/terminal.rs": { + "hooks": 1, + "tests": 4 + }, + "nvx-rs/src/guest/time/api.rs": { + "hooks": 3, + "tests": 17 + }, + "nvx-rs/src/guest/time/checks.rs": { + "hooks": 1, + "tests": 6 + }, + "nvx-rs/src/guest/time/commands.rs": { + "hooks": 1, + "tests": 3 + }, + "nvx-rs/src/guest/time/cpu_mask.rs": { + "hooks": 1, + "tests": 3 + }, + "nvx-rs/src/guest/time/daemon.rs": { + "hooks": 1, + "tests": 9 + }, + "nvx-rs/src/guest/time/decoders.rs": { + "hooks": 1, + "tests": 28 + }, + "nvx-rs/src/guest/time/events.rs": { + "hooks": 1, + "tests": 3 + }, + "nvx-rs/src/guest/time/mod.rs": { + "hooks": 1, + "tests": 2 + }, + "nvx-rs/src/guest/time/state.rs": { + "hooks": 1, + "tests": 8 + }, + "nvx-rs/src/guest/time/system.rs": { + "hooks": 4, + "tests": 25 + }, + "nvx-rs/src/guest/time/test_mode.rs": { + "hooks": 1, + "tests": 10 + }, + "nvx-rs/src/guest/time_probe/clock.rs": { + "hooks": 1, + "tests": 6 + }, + "nvx-rs/src/guest/time_probe/format.rs": { + "hooks": 1, + "tests": 3 + }, + "nvx-rs/src/guest/time_probe/latency.rs": { + "hooks": 1, + "tests": 1 + }, + "nvx-rs/src/guest/time_probe/mod.rs": { + "hooks": 1, + "tests": 2 + }, + "nvx-rs/src/guest/time_probe/support.rs": { + "hooks": 1, + "tests": 6 + }, + "nvx-rs/src/guest/time_probe/warp.rs": { + "hooks": 1, + "tests": 4 + }, + "nvx-rs/src/host/cli/args.rs": { + "hooks": 2, + "tests": 10 + }, + "nvx-rs/src/host/cli/artifacts.rs": { + "hooks": 1, + "tests": 1 + }, + "nvx-rs/src/host/cli/cpu_profile.rs": { + "hooks": 1, + "tests": 9 + }, + "nvx-rs/src/host/cli/deadline.rs": { + "hooks": 1, + "tests": 1 + }, + "nvx-rs/src/host/cli/download/github.rs": { + "hooks": 1, + "tests": 0 + }, + "nvx-rs/src/host/cli/json.rs": { + "hooks": 1, + "tests": 2 + }, + "nvx-rs/src/host/cli/mod.rs": { + "hooks": 1, + "tests": 5 + }, + "nvx-rs/src/host/cli/output.rs": { + "hooks": 1, + "tests": 2 + }, + "nvx-rs/src/host/cli/run.rs": { + "hooks": 1, + "tests": 0 + }, + "nvx-rs/src/host/cli/run/specification.rs": { + "hooks": 0, + "tests": 4 + }, + "nvx-rs/src/host/cli/sandbox/lock.rs": { + "hooks": 1, + "tests": 5 + }, + "nvx-rs/src/host/cli/sandbox/mod.rs": { + "hooks": 1, + "tests": 0 + }, + "nvx-rs/src/host/cli/shell.rs": { + "hooks": 1, + "tests": 2 + }, + "nvx-rs/src/host/launch.rs": { + "hooks": 1, + "tests": 16 + }, + "nvx-rs/src/host/mod.rs": { + "hooks": 1, + "tests": 0 + }, + "nvx-rs/src/host/network/address.rs": { + "hooks": 4, + "tests": 6 + }, + "nvx-rs/src/host/network/lowering.rs": { + "hooks": 1, + "tests": 3 + }, + "nvx-rs/src/host/network/policy/json.rs": { + "hooks": 1, + "tests": 7 + }, + "nvx-rs/src/host/paths.rs": { + "hooks": 1, + "tests": 6 + }, + "nvx-rs/src/host/platform/linux/mod.rs": { + "hooks": 2, + "tests": 0 + }, + "nvx-rs/src/host/platform/linux/process.rs": { + "hooks": 1, + "tests": 0 + }, + "nvx-rs/src/host/platform/mod.rs": { + "hooks": 1, + "tests": 3 + }, + "nvx-rs/src/host/platform/windows/mod.rs": { + "hooks": 1, + "tests": 0 + }, + "nvx-rs/src/host/platform/windows/private.rs": { + "hooks": 1, + "tests": 0 + }, + "nvx-rs/src/host/process.rs": { + "hooks": 2, + "tests": 15 + }, + "nvx-rs/src/host/report.rs": { + "hooks": 1, + "tests": 1 + }, + "nvx-rs/src/host/sandbox/layer.rs": { + "hooks": 1, + "tests": 3 + }, + "nvx-rs/src/host/sandbox/mod.rs": { + "hooks": 1, + "tests": 0 + }, + "nvx-rs/src/host/sandbox/mount/mod.rs": { + "hooks": 1, + "tests": 0 + }, + "nvx-rs/src/host/sandbox/specification.rs": { + "hooks": 0, + "tests": 22 + }, + "nvx-rs/src/host/seal.rs": { + "hooks": 2, + "tests": 0 + }, + "nvx-rs/src/host/seal/windows.rs": { + "hooks": 2, + "tests": 3 + }, + "nvx-rs/src/host/session.rs": { + "hooks": 1, + "tests": 16 + }, + "nvx-rs/src/host/spawn.rs": { + "hooks": 1, + "tests": 7 + }, + "nvx-rs/src/host/store/caller.rs": { + "hooks": 1, + "tests": 6 + }, + "nvx-rs/src/host/sys/mod.rs": { + "hooks": 1, + "tests": 0 + }, + "nvx-rs/src/host/sys/process.rs": { + "hooks": 3, + "tests": 0 + }, + "nvx-rs/src/host/template/canonical.rs": { + "hooks": 1, + "tests": 2 + }, + "nvx-rs/src/host/template/clone/failure.rs": { + "hooks": 1, + "tests": 10 + }, + "nvx-rs/src/host/template/clone/launch.rs": { + "hooks": 1, + "tests": 0 + }, + "nvx-rs/src/host/template/clone/records.rs": { + "hooks": 1, + "tests": 11 + }, + "nvx-rs/src/host/template/mod.rs": { + "hooks": 1, + "tests": 0 + }, + "nvx-rs/src/host/template/store.rs": { + "hooks": 2, + "tests": 0 + }, + "nvx-rs/src/host/template/wire.rs": { + "hooks": 1, + "tests": 6 + }, + "nvx-rs/src/hypervisor.rs": { + "hooks": 1, + "tests": 2 + }, + "nvx-rs/src/net/cidr.rs": { + "hooks": 1, + "tests": 7 + }, + "nvx-rs/src/net/mod.rs": { + "hooks": 1, + "tests": 3 + }, + "nvx-rs/src/protocol/cmdline.rs": { + "hooks": 1, + "tests": 0 + }, + "nvx-rs/src/protocol/nvxs/record.rs": { + "hooks": 1, + "tests": 0 + }, + "nvx-rs/src/protocol/share.rs": { + "hooks": 1, + "tests": 2 + }, + "nvx-rs/src/protocol/status.rs": { + "hooks": 1, + "tests": 2 + }, + "nvx-rs/src/testing/fake_openvmm.rs": { + "hooks": 1, + "tests": 13 + }, + "nvx-rs/src/uuid.rs": { + "hooks": 1, + "tests": 4 + }, + "tests/runtime-adapter/src/frame.rs": { + "hooks": 1, + "tests": 8 + }, + "tests/runtime-adapter/src/generations.rs": { + "hooks": 1, + "tests": 2 + }, + "tests/runtime-adapter/src/probe.rs": { + "hooks": 1, + "tests": 4 + }, + "tests/runtime-adapter/src/root.rs": { + "hooks": 1, + "tests": 10 + }, + "tests/runtime-adapter/src/template.rs": { + "hooks": 1, + "tests": 9 + }, + "tests/runtime-adapter/src/worker/activation.rs": { + "hooks": 1, + "tests": 3 + }, + "tests/runtime-adapter/src/worker/capture.rs": { + "hooks": 1, + "tests": 7 + }, + "tests/runtime-adapter/src/worker/entries.rs": { + "hooks": 1, + "tests": 6 + }, + "tests/runtime-adapter/src/worker/mapping.rs": { + "hooks": 1, + "tests": 2 + }, + "tests/runtime-adapter/src/worker/probes.rs": { + "hooks": 1, + "tests": 10 + } + }, + "seams": { + "nvx-rs/src/guest/clone/activation/apply.rs": 1, + "nvx-rs/src/host/platform/windows/spawn.rs": 1, + "nvx-rs/src/host/sandbox/plan.rs": 5 + } +} diff --git a/scripts/test_test_changes.py b/scripts/test_test_changes.py new file mode 100644 index 000000000..8ef00b9ec --- /dev/null +++ b/scripts/test_test_changes.py @@ -0,0 +1,1602 @@ +"""Tests of check_test_changes.py, which flags the changes to existing tests.""" + +from __future__ import annotations + +import contextlib +import io +import json +import subprocess +import sys +import tempfile +import unittest +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import cast + +import yaml + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import check_test_changes as changes # noqa: E402 +import check_test_layout as layout # noqa: E402 + +CARGO = '[package]\nname = "demo"\nversion = "0.1.0"\n' +WORKFLOW = layout.REPO_ROOT / ".github" / "workflows" / "test-changes.yml" + + +def flagged(found: Sequence[changes.Change]) -> list[tuple[str, str, str]]: + return sorted( + (change.category, change.action, change.kind) + for change in found + if change.flags + ) + + +def ways(*predicates: set[str]) -> changes.Conditions: + """Returns the ways that code compiles, one for each set of predicates.""" + return frozenset(frozenset(way) for way in predicates) + + +ANYWHERE = ways(set()) + + +def actions(found: Sequence[changes.Change]) -> list[str]: + return sorted(change.action for change in found) + + +def rust( + base: Mapping[str, str], head: Mapping[str, str], crate: str = "demo" +) -> list[changes.Change]: + """Compares the test entries of Rust files of a crate, whose modules + inherit the cfg predicates of their declarations among the files.""" + + def entries(files: Mapping[str, str]) -> list[changes.Entry]: + full = {f"{crate}/{path}": text for path, text in files.items()} + # The crate's root declares the top-level modules, unless given. + root = f"{crate}/src/lib.rs" + names = sorted( + { + path.split("/")[2].removesuffix(".rs") + for path in full + if path.startswith(f"{crate}/src/") and path != root + } + ) + tree = {root: "".join(f"mod {name};\n" for name in names), **full} + inherited = changes.module_conditions( + tree.keys(), + lambda paths: {path: tree[path] for path in paths}, + (crate,), + full, + ) + found: list[changes.Entry] = [] + for path, text in full.items(): + role = layout.rust_role(path, crate) + found.extend( + changes.rust_entries( + path, + crate, + text, + changes.TEST_CODE, + role == "test", + # Files that no root among these reaches still compare. + inherited[path] or changes.ANYWHERE, + ) + ) + return found + + return changes.compare(entries(base), entries(head)) + + +def python(base: Mapping[str, str], head: Mapping[str, str]) -> list[changes.Change]: + def entries(files: Mapping[str, str]) -> list[changes.Entry]: + found: list[changes.Entry] = [] + for path, text in files.items(): + found.extend(changes.python_entries(path, text, changes.TEST_CODE)) + return found + + return changes.compare(entries(base), entries(head)) + + +INLINE = """pub fn f() -> u8 { + 1 +} + +/// The tests of `f`. +#[cfg(test)] +mod tests { + use super::*; + + const EXPECTED: u8 = 1; + + #[test] + fn works() { + assert_eq!(f(), EXPECTED); + } +} +""" +DECLARED = """pub fn f() -> u8 { + 1 +} + +/// The tests of `f`. +#[cfg(test)] +mod tests; +""" +MOVED = """//! The tests of `f`. + +use super::*; + +const EXPECTED: u8 = 1; + +#[test] +fn works() { + // The value that `f` documents. + assert_eq!(f(), EXPECTED); +} +""" + + +class CompareTests(unittest.TestCase): + def entry( + self, + key: str, + content: str, + moved: str | None = None, + name: str = "", + kind: str = "test", + path: str = "a.rs", + monotone: bool = False, + ) -> changes.Entry: + return changes.Entry( + ("rust", key), + ("rust", "fn", name or key), + ("rust", "scope"), + name or key, + content, + content if moved is None else moved, + changes.TEST_CODE, + kind, + f"`{key}`", + changes.Location(path, 1), + monotone, + ) + + def test_entries_change_by_identity_then_by_content(self) -> None: + base = [ + self.entry("same", "a b c"), + self.entry("edited", "a b c"), + self.entry("moved", "x y z", path="old.rs"), + self.entry("relocated", "p q r", moved="P"), + self.entry("gone", "1 2 3"), + self.entry("imports", "a\nb", kind="imports", monotone=True), + self.entry("pruned", "a\nb", kind="imports", monotone=True), + ] + head = [ + self.entry("same", "a b c"), + self.entry("edited", "a b d"), + self.entry("moved", "x y z", path="new.rs"), + self.entry("elsewhere", "other", moved="P", name="relocated"), + self.entry("imports", "a\nb\nc", kind="imports", monotone=True), + self.entry("pruned", "a", kind="imports", monotone=True), + self.entry("new", "n"), + ] + found = {change.what: change.action for change in changes.compare(base, head)} + self.assertEqual( + found, + { + "`edited`": "modified", + "`moved`": "relocated", + "`relocated`": "relocated", + "`gone`": "removed", + "`imports`": "extended", + "`pruned`": "modified", + "`new`": "added", + }, + ) + + def test_a_removed_entry_pairs_with_a_new_one_of_its_name_or_content(self) -> None: + base = [ + self.entry("old", "assert eq f 1"), + self.entry("renamed", "alpha beta gamma delta epsilon zeta"), + ] + head = [ + self.entry("other", "assert eq f 2", name="old"), + self.entry("rename", "alpha beta gamma delta epsilon theta"), + ] + found = { + change.what: (change.action, change.detail) + for change in changes.compare(base, head) + } + self.assertEqual( + found, + { + "`old`": ("modified", "moved"), + "`renamed`": ("modified", "renamed to `rename`"), + }, + ) + + +class RustEntryTests(unittest.TestCase): + def test_an_inline_tests_module_moves_to_its_own_file_unchanged(self) -> None: + found = rust( + {"src/a.rs": INLINE}, + {"src/a.rs": DECLARED, "src/a/tests.rs": MOVED}, + ) + self.assertEqual(flagged(found), []) + self.assertEqual(actions(found), ["relocated"] * 4) + + def test_changes_to_what_tests_do_count(self) -> None: + for edited, count in ( + (MOVED.replace("#[test]", "#[test]\n#[ignore]"), 1), + (MOVED.replace("EXPECTED);", "EXPECTED + 0);"), 1), + (MOVED.replace("const EXPECTED: u8 = 1;", "const EXPECTED: u8 = 2;"), 1), + (MOVED.replace("use super::*;", "use super::f;"), 1), + # The attribute, and where each item compiles. + ("#![cfg(unix)]\n" + MOVED, 4), + ): + with self.subTest(edited=edited): + found = rust( + {"src/a.rs": DECLARED, "src/a/tests.rs": MOVED}, + {"src/a.rs": DECLARED, "src/a/tests.rs": edited}, + ) + self.assertEqual(len(flagged(found)), count) + + def test_formatting_comments_lints_and_visibility_do_not_count(self) -> None: + edited = ( + MOVED.replace("const EXPECTED", "pub(crate) const EXPECTED") + .replace("#[test]", "#[test]\n#[allow(clippy::unit_cmp)]") + .replace( + "assert_eq!(f(), EXPECTED);", + "assert_eq!(\n f(),\n EXPECTED, // f\n);", + ) + .replace("use super::*;", "#![allow(dead_code)]\nuse super::*;") + ) + found = rust( + {"src/a.rs": DECLARED, "src/a/tests.rs": MOVED}, + {"src/a.rs": DECLARED, "src/a/tests.rs": edited}, + ) + self.assertEqual(found, []) + + def test_commas_count_only_where_they_mean_something(self) -> None: + def test(body: str) -> str: + return f"#[test]\nfn t() {{\n {body}\n}}\n" + + for before, after, counts in ( + ("g(a, b);", "g(\n a,\n b,\n );", False), + ("let v = [1, 2];", "let v = [1, 2,];", False), + ("match x { A => { 1 }, B => 2 }", "match x { A => { 1 } B => 2, }", False), + ( + "match x { A => { f(1) } B => 2 }", + "match x { A => f(1), B => 2 }", + False, + ), + ("match x { A => { f(1) } }", "match x { A => { let y = 1; f(y) } }", True), + ("match x { A => f(1), B => 2 }", "match x { A => f(2), B => 2 }", True), + ("v.map(|x| { x + 1 });", "v.map(|x| x + 1);", False), + ("v.map(|x| { x + 1 });", "v.map(|x| x + 2);", True), + ("v.map(|x| -> u8 { x });", "v.map(|x| -> u8 { x + 1 });", True), + ("let t = (x,);", "let t = (x);", True), + ("let t = ({ 1 },);", "let t = ({ 1 });", True), + ("let t = (x, y);", "let t = (x, y,);", False), + ("return (x,);", "return (x);", True), + ("let v = [{ 1 }, -2];", "let v = [{ 1 } -2];", True), + # A macro may read a comma or a brace as it chooses. + ("check!(1);", "check!(1,);", True), + ("check!(g(a, b));", "check!(g(a, b,));", True), + ("check! { a => { 1 } }", "check! { a => 1 }", True), + ( + "macro_rules! m { ($a:expr) => { $a }; }", + "macro_rules! m { ($a:expr,) => { $a }; }", + True, + ), + ( + "check!(match x { A => { 1 } B => 2 });", + "check!(match x { A => 1, B => 2, });", + False, + ), + # The standard library's take expressions and a trailing comma. + ( + "assert_eq!(g(a, b), c);", + "assert_eq!(\n g(a, b,),\n c,\n );", + False, + ), + ("let v = vec![1, 2];", "let v = vec![1, 2,];", False), + ): + with self.subTest(before=before): + found = rust( + {"src/a/tests.rs": test(before)}, {"src/a/tests.rs": test(after)} + ) + self.assertEqual(bool(flagged(found)), counts) + + def test_test_helpers_move_into_the_tests_module(self) -> None: + base = { + "src/a.rs": "pub struct S {\n x: u8,\n}\n\nimpl S {\n" + " #[cfg(test)]\n pub(crate) fn x(&self) -> u8 {\n self.x\n }\n}\n" + "\n#[cfg(all(test, unix))]\npub(crate) fn probe() -> u8 {\n 1\n}\n", + } + head = { + "src/a.rs": "pub struct S {\n x: u8,\n}\n\n#[cfg(test)]\nmod tests;\n", + "src/a/tests.rs": "use super::*;\n\nimpl S {\n fn x(&self) -> u8 {\n" + " self.x\n }\n}\n\n#[cfg(unix)]\nfn probe() -> u8 {\n 1\n}\n", + } + found = rust(base, head) + self.assertEqual(flagged(found), []) + self.assertIn("relocated", actions(found)) + + def test_includes_follow_the_file_that_moves(self) -> None: + inline = INLINE.replace( + "const EXPECTED: u8 = 1;", + 'const EXPECTED: &str = include_str!("../README.md");', + ) + moved = MOVED.replace( + "const EXPECTED: u8 = 1;", + 'const EXPECTED: &str = include_str!("../../README.md");', + ) + found = rust( + {"src/a.rs": inline}, + {"src/a.rs": DECLARED, "src/a/tests.rs": moved}, + ) + self.assertEqual(flagged(found), []) + renamed = moved.replace("../../README.md", "../../CHANGES.md") + found = rust( + {"src/a.rs": inline}, + {"src/a.rs": DECLARED, "src/a/tests.rs": renamed}, + ) + self.assertEqual(flagged(found), [(changes.TEST_CODE, "modified", "helper")]) + + def test_test_hooks_count_but_production_code_does_not(self) -> None: + base = ( + "pub fn f(x: u8) -> u8 {\n" + " #[cfg(test)]\n" + " if let Some(value) = take() {\n" + " return value;\n" + " }\n" + " x + 1\n" + "}\n" + ) + hook = base.replace("return value;", "return value + 1;") + production = base.replace("x + 1", "x + 2") + self.assertEqual( + flagged(rust({"src/a.rs": base}, {"src/a.rs": hook})), + [(changes.TEST_CODE, "modified", "hook")], + ) + self.assertEqual(rust({"src/a.rs": base}, {"src/a.rs": production}), []) + + def test_hooks_of_conditions_end_with_their_bodies(self) -> None: + base = ( + "pub fn f() -> u8 {\n" + " #[cfg(test)]\n" + " if let Some(value) = { take() } {\n" + " return value;\n" + " }\n" + " #[cfg(test)]\n" + " for S { a } in { items() } {\n" + " record(a);\n" + " }\n" + " #[cfg(test)]\n" + " while let 1..=5 = { next() } {\n" + " pause();\n" + " }\n" + " 1\n" + "}\n" + ) + for old, new in ( + ("return value;", "return value + 1;"), + ("record(a);", "record(a + 1);"), + ("pause();", "pause_more();"), + ): + with self.subTest(old=old): + self.assertEqual( + flagged( + rust({"src/a.rs": base}, {"src/a.rs": base.replace(old, new)}) + ), + [(changes.TEST_CODE, "modified", "hook")], + ) + + def test_hooks_end_with_their_construct(self) -> None: + source = ( + "fn f(kind: Kind) -> u8 {\n" + " #[cfg(test)]\n" + " record!(kind);\n" + " #[cfg(test)]\n" + " let seen: HashMap = HashMap::new();\n" + " #[cfg(test)]\n" + " let add = |a, b| a + b;\n" + " match kind {\n" + " #[cfg(test)]\n" + " Kind::Fake => { 1 }\n" + " #[cfg(test)]\n" + " Kind::Probe => convert::(1),\n" + " Kind::Real => 2,\n" + " }\n" + "}\n" + "fn g(a: u8, b: u8) -> S {\n" + " S {\n" + " #[cfg(test)]\n" + " check: |a, b| a < b,\n" + " #[cfg(test)]\n" + " less: a < b,\n" + " more: a > b,\n" + " }\n" + "}\n" + "struct S {\n" + " #[cfg(test)]\n" + " seen: u8,\n" + " #[cfg(test)]\n" + " probe: Result,\n" + " other: u8,\n" + "}\n" + ) + entries = changes.rust_entries( + "demo/src/a.rs", "demo", source, changes.TEST_CODE, False + ) + hooks = {entry.name: entry.content for entry in entries} + self.assertEqual( + hooks, + { + "fn f": "# [ cfg ( test ) ] record ! ( kind ) ; | " + "# [ cfg ( test ) ] let seen : HashMap < u8 , u16 > = HashMap :: new " + "( ) ; | # [ cfg ( test ) ] let add = | a , b | a + b ; | " + "# [ cfg ( test ) ] Kind :: Fake => 1 | " + "# [ cfg ( test ) ] Kind :: Probe => convert :: < u8 , u16 > ( 1 ) ,", + "fn g": "# [ cfg ( test ) ] check : | a , b | a < b , | " + "# [ cfg ( test ) ] less : a < b ,", + "struct S": "# [ cfg ( test ) ] seen : u8 , | " + "# [ cfg ( test ) ] probe : Result < u8 , u16 > ,", + }, + ) + + def test_operators_keep_their_punctuation_together(self) -> None: + files = {"src/a.rs": DECLARED} + base = "#[test]\nfn t() {\n assert!(ready() && check());\n}\n" + split = base.replace("ready() && check()", "ready() & &check()") + self.assertEqual( + flagged( + rust( + {**files, "src/a/tests.rs": base}, + {**files, "src/a/tests.rs": split}, + ) + ), + [(changes.TEST_CODE, "modified", "test")], + ) + # rustfmt breaks a chain of calls that grows too wide, which changes + # nothing. + chained = ( + "#[test]\nfn t() -> Result<(), E> {\n let v = make()?.value();\n" + " Ok(())\n}\n" + ) + wrapped = chained.replace("make()?.value()", "make()?\n .value()") + self.assertEqual( + flagged( + rust( + {**files, "src/a/tests.rs": chained}, + {**files, "src/a/tests.rs": wrapped}, + ) + ), + [], + ) + + def test_commas_and_braces_that_carry_meaning_count(self) -> None: + files = {"src/a.rs": DECLARED} + + def compare(base: str, head: str) -> list[tuple[str, str, str]]: + return flagged( + rust( + {**files, "src/a/tests.rs": base}, {**files, "src/a/tests.rs": head} + ) + ) + + # After `->` or `=>`, a comma makes a tuple of one. + returns = "fn accepts(_: impl Fn() -> (u8,)) {}\n" + self.assertEqual( + compare(returns, returns.replace("(u8,)", "(u8)")), + [(changes.TEST_CODE, "modified", "helper")], + ) + arm = "fn pick(x: u8) -> (u8,) {\n match x {\n _ => (x,),\n }\n}\n" + self.assertEqual( + compare(arm, arm.replace("_ => (x,)", "_ => (x)")), + [(changes.TEST_CODE, "modified", "helper")], + ) + # A block after an operator `|` keeps its braces, and a closure's body + # does not. + operand = "#[test]\nfn t() {\n assert!(true | { true == false });\n}\n" + self.assertEqual( + compare(operand, operand.replace("{ true == false }", "true == false")), + [(changes.TEST_CODE, "modified", "test")], + ) + closure = "#[test]\nfn t() {\n assert!(v.iter().all(|x| { *x > 0 }));\n}\n" + self.assertEqual(compare(closure, closure.replace("{ *x > 0 }", "*x > 0")), []) + + def test_only_paths_changed(self) -> None: + edited = MOVED.replace("assert_eq!(f()", "assert_eq!(super::f()") + found = rust( + {"src/a.rs": DECLARED, "src/a/tests.rs": MOVED}, + {"src/a.rs": DECLARED, "src/a/tests.rs": edited}, + ) + self.assertEqual( + [change.detail for change in found if change.flags], + ["only the paths that it names changed"], + ) + + def test_proptest_functions_and_platform_twins_are_tests(self) -> None: + source = ( + "proptest! {\n" + " #![proptest_config(ProptestConfig::with_cases(8))]\n" + " #[test]\n" + " fn holds(x in 0..10u8) {\n" + " prop_assert!(x < 10);\n" + " }\n" + "}\n" + "#[cfg(unix)]\n#[test]\nfn twin() {}\n" + "#[cfg(windows)]\n#[test]\nfn twin() {}\n" + "#[cfg_attr(unix, test)]\nfn applied() {}\n" + ) + entries = changes.rust_entries( + "demo/src/a/tests.rs", "demo", source, changes.TEST_CODE, True + ) + self.assertEqual( + sorted((entry.name, entry.kind) for entry in entries), + [ + ("", "attributes"), + ("applied", "test"), + ("holds", "test"), + ("proptest", "helper"), + ("twin", "test"), + ("twin", "test"), + ], + ) + self.assertEqual(len({entry.key for entry in entries}), 6) + + def test_crate_tests_are_modules_of_their_own(self) -> None: + self.assertEqual( + changes.rust_module("demo/tests/cli/harness.rs", "demo"), + ("[tests]", "cli", "harness"), + ) + self.assertEqual( + changes.rust_module("demo/src/a/tests.rs", "demo"), ("a", "tests") + ) + + def test_a_move_keeps_where_what_moves_compiles(self) -> None: + test = ( + "#[test]\nfn works() {\n // The value that `f` documents.\n" + " assert_eq!(f(), EXPECTED);\n}\n" + ) + wrapped = MOVED.replace( + test, + "#[cfg(any())]\nmod moved {\n use super::*;\n\n" + " #[test]\n fn works() {\n assert_eq!(f(), EXPECTED);\n" + " }\n}\n", + ) + found = rust( + {"src/a.rs": DECLARED, "src/a/tests.rs": MOVED}, + {"src/a.rs": DECLARED, "src/a/tests.rs": wrapped}, + ) + self.assertEqual( + [ + (change.action, change.detail) + for change in found + if change.kind == "test" + ], + [ + ( + "modified", + "moved, and it now compiles where `any ( )` and `test` " + "hold, not where `test` did", + ) + ], + ) + base = { + "src/a.rs": DECLARED, + "src/a/tests.rs": MOVED, + "src/b.rs": "pub fn g() {}\n", + } + for declaration, flags in ( + ("#[cfg(test)]\nmod tests;\n", False), + ("#[cfg(all(test, windows))]\nmod tests;\n", True), + ): + with self.subTest(declaration=declaration): + head = { + "src/a.rs": "pub fn f() -> u8 {\n 1\n}\n", + "src/b.rs": "pub fn g() {}\n\n" + declaration, + "src/b/tests.rs": MOVED, + } + found = rust(base, head) + self.assertEqual( + (changes.TEST_CODE, "modified", "test") in flagged(found), flags + ) + + def test_modules_inherit_the_conditions_of_their_declarations(self) -> None: + files = { + "demo/src/lib.rs": "#[cfg(unix)]\nmod a;\n" + "#[cfg_attr(windows, cfg_attr(test, cfg(any())))]\nmod c;\n", + "demo/src/a.rs": '#![cfg(feature = "x")]\nmod inner {\n' + ' #![cfg(target_os = "linux")]\n mod b;\n}\n' + "#[cfg(test)]\nmod tests;\n", + "demo/src/a/inner/b.rs": "", + "demo/src/a/tests.rs": "", + "demo/src/c.rs": "", + "demo/src/orphan.rs": "#[cfg(test)]\nmod tests;\n", + "demo/src/orphan/tests.rs": "", + "demo/tests/cli.rs": '#[cfg(windows)]\n#[path = "cli/harness.rs"]\n' + "mod harness;\nmod common;\n", + "demo/tests/cli/harness.rs": "", + "demo/tests/common/mod.rs": "", + } + found = changes.module_conditions( + files.keys(), + lambda paths: {path: files[path] for path in paths}, + ("demo",), + files, + ) + self.assertEqual( + found, + { + "demo/src/lib.rs": ANYWHERE, + "demo/src/a.rs": ways({"unix"}), + "demo/src/a/inner/b.rs": ways( + {"unix", 'feature = "x"', 'target_os = "linux"'} + ), + "demo/src/a/tests.rs": ways({"unix", 'feature = "x"', "test"}), + "demo/src/c.rs": ways( + {"any ( not ( all ( windows , test ) ) , any ( ) )"} + ), + "demo/src/orphan.rs": None, + "demo/src/orphan/tests.rs": None, + "demo/tests/cli.rs": ANYWHERE, + "demo/tests/cli/harness.rs": ways({"windows"}), + "demo/tests/common/mod.rs": ANYWHERE, + }, + ) + + def test_targets_whose_tests_never_run_reach_no_module(self) -> None: + files = { + "demo/Cargo.toml": '[package]\nname = "demo"\n\n[[bin]]\nname = "tool"\n' + "test = false\n", + "demo/src/lib.rs": "", + "demo/src/bin/tool/main.rs": "mod a;\n", + "demo/src/bin/tool/a.rs": "", + "demo/benches/speed.rs": "mod b;\n", + "demo/benches/b.rs": "", + } + found = changes.module_conditions( + files.keys(), + lambda paths: {path: files[path] for path in paths}, + ("demo",), + [path for path in files if path.endswith(".rs")], + ) + self.assertEqual( + found, + { + "demo/src/lib.rs": ANYWHERE, + "demo/src/bin/tool/main.rs": None, + "demo/src/bin/tool/a.rs": None, + "demo/benches/speed.rs": None, + "demo/benches/b.rs": None, + }, + ) + + def test_modules_reach_through_path_attributes_and_other_targets(self) -> None: + files = { + "demo/Cargo.toml": CARGO + '\n[[test]]\nname = "common"\ntest = false\n', + "demo/src/lib.rs": "", + "demo/tests/cli.rs": "mod suite;\nmod common;\n", + "demo/tests/suite/mod.rs": '#[path = "../shared/cases.rs"]\nmod cases;\n', + "demo/tests/shared/cases.rs": "", + # A root of a target whose tests never run, which cli.rs compiles. + "demo/tests/common.rs": "", + } + found = changes.module_conditions( + files.keys(), + lambda paths: {path: files[path] for path in paths}, + ("demo",), + ["demo/tests/shared/cases.rs", "demo/tests/common.rs"], + ) + self.assertEqual( + found, + { + "demo/tests/shared/cases.rs": ANYWHERE, + "demo/tests/common.rs": ANYWHERE, + }, + ) + + def test_modules_that_inner_attributes_guard_hold_test_code(self) -> None: + base = ( + "pub fn f() {}\n\nmod checks {\n #![cfg(test)]\n\n" + " #[test]\n fn works() {\n assert!(true);\n }\n}\n" + ) + edited = base.replace("assert!(true);", "assert!(false);") + self.assertEqual( + flagged(rust({"src/a.rs": base}, {"src/a.rs": edited})), + [(changes.TEST_CODE, "modified", "test")], + ) + whole = "#![cfg(test)]\n\n#[test]\nfn works() {\n assert!(true);\n}\n" + self.assertEqual( + flagged( + rust({"src/b.rs": whole}, {"src/b.rs": whole.replace("true", "false")}) + ), + [(changes.TEST_CODE, "modified", "test")], + ) + + def test_tests_and_helpers_that_no_cfg_test_guards_count(self) -> None: + body = "\nfn works() {\n assert!(true);\n}\n" + for attribute in ( + "#[test]", + "#[cfg_attr(unix, test)]", + "#[cfg_attr(all(), cfg(test))]", + ): + with self.subTest(attribute=attribute): + source = f"pub fn f() {{}}\n\n{attribute}{body}" + edited = source.replace("assert!(true);", "assert!(false);") + self.assertEqual( + len(flagged(rust({"src/a.rs": source}, {"src/a.rs": edited}))), 1 + ) + + def test_the_attributes_of_an_import_count(self) -> None: + base = MOVED.replace( + "use super::*;\n", + "use super::*;\n#[cfg(unix)]\nuse std::os::unix::fs::PermissionsExt;\n", + ) + for edited, expected in ( + (base.replace("#[cfg(unix)]", "#[cfg(windows)]"), ["modified"]), + ( + base.replace("#[cfg(unix)]", "#[allow(unused_imports)]\n#[cfg(unix)]"), + [], + ), + (base.replace("#[cfg(unix)]\n", ""), ["modified"]), + ): + with self.subTest(edited=edited): + found = rust( + {"src/a.rs": DECLARED, "src/a/tests.rs": base}, + {"src/a.rs": DECLARED, "src/a/tests.rs": edited}, + ) + self.assertEqual( + [change.action for change in found if change.flags], expected + ) + + +class PythonEntryTests(unittest.TestCase): + BASE = '''"""Tests.""" + +import unittest +from pathlib import Path + + +class ATests(unittest.TestCase): + def test_a(self): + """Checks a.""" + self.assertEqual(1, 1) + + def helper(self): + return Path(".") + + +if __name__ == "__main__": + unittest.main() +''' + + def test_methods_imports_and_moves(self) -> None: + path = "scripts/test_a.py" + docstring = self.BASE.replace("Checks a.", "Checks that a holds.") + self.assertEqual(python({path: self.BASE}, {path: docstring}), []) + edited = self.BASE.replace("assertEqual(1, 1)", "assertTrue(True)") + self.assertEqual( + flagged(python({path: self.BASE}, {path: edited})), + [(changes.TEST_CODE, "modified", "test")], + ) + extended = self.BASE.replace( + "import unittest\n", "import json\nimport unittest\n" + ) + self.assertEqual( + actions(python({path: self.BASE}, {path: extended})), ["extended"] + ) + reduced = self.BASE.replace("from pathlib import Path\n", "") + self.assertEqual( + flagged(python({path: self.BASE}, {path: reduced})), + [(changes.TEST_CODE, "modified", "imports")], + ) + moved = self.BASE.replace("class ATests", "class BTests") + found = python({path: self.BASE}, {"scripts/test_b.py": moved}) + self.assertEqual(flagged(found), [(changes.TEST_CODE, "removed", "helper")]) + self.assertEqual(actions(found).count("relocated"), 4) + + +class WiringTests(unittest.TestCase): + def test_changed_lines_of_test_actions(self) -> None: + base = ( + "steps:\n - name: Test\n run: |\n" + " cargo test --locked\n cargo test --lib\n# note\n" + ) + for head, expected in ( + (base.replace("# note", "# other note"), []), + (base.replace(" cargo test --lib\n", ""), ["removed"]), + (base.replace("--locked", "--locked --exact"), ["modified"]), + (base + " cargo test --doc\n", []), + ( + base.replace(" run: |", " if: false\n run: |"), + ["added a line that may skip"], + ), + (base + " continue-on-error: true\n", ["added a line that may skip"]), + # A conditional that wraps a test command keeps the command's line. + ( + base.replace( + " cargo test --locked\n", + " if false; then\n cargo test --locked\n fi\n", + ), + ["added a line that may skip", "added a line that may skip"], + ), + ( + base.replace( + " cargo test --lib\n", + " If ($false) {\n cargo test --lib\n }\n", + ), + ["added a line that may skip"], + ), + ): + with self.subTest(head=head): + found = changes.line_changes("action.yml", base, head) + self.assertEqual([change.action for change in found], expected) + nested = base.replace(" run: |", " env:\n if: 'false'\n run: |") + dedented = nested.replace(" if: 'false'", " if: 'false'") + self.assertEqual( + [ + change.action + for change in changes.line_changes("action.yml", nested, dedented) + ], + ["modified", "added a line that may skip"], + ) + self.assertEqual(changes.line_changes("action.yml", None, base), []) + self.assertEqual( + [ + change.action + for change in changes.line_changes("action.yml", base, None) + ], + ["removed"], + ) + + def test_tokens_that_ci_names(self) -> None: + base = ( + " - uses: ./.github/actions/validate-nvx\n" + " run: >-\n" + " python3 scripts/nvx.py test-microvm --scenario guest-boot\n" + ' cargo "+${{ steps.t.outputs.toolchain }}" test --locked \\\n' + " --all-features\n" + " python -m unittest scripts\\test_a.py\n" + ) + tokens = changes.wiring_tokens(base) + self.assertEqual( + sorted(tokens), + [ + "0:./.github/actions/validate-nvx", + "1:test-microvm", + "2:guest-boot", + "3:test_a.py", + "4:cargo test --locked --all-features", + ], + ) + self.assertEqual(tokens["4:cargo test --locked --all-features"], [4]) + head = base.replace("--scenario guest-boot", "").replace( + " --all-features", " --no-default-features" + ) + self.assertEqual( + [change.what for change in changes.token_changes("ci.yml", base, head)], + ["`guest-boot`", "`cargo test --locked --all-features`"], + ) + + def test_a_token_counts_each_time_that_ci_names_it(self) -> None: + job = " - uses: ./.github/actions/validate-nvx\n" + both = f"linux:\n{job}windows:\n{job}" + self.assertEqual( + changes.wiring_tokens(both)["0:./.github/actions/validate-nvx"], [2, 4] + ) + found = changes.token_changes("ci.yml", both, f"linux:\n{job}") + self.assertEqual( + [ + (change.action, cast(changes.Location, change.base).line) + for change in found + ], + [("removed", 4)], + ) + commented = f"linux:\n{job}windows:\n # {job.strip()}\n" + self.assertEqual( + [ + change.action + for change in changes.token_changes("ci.yml", both, commented) + ], + ["removed"], + ) + + def test_jobs_that_run_tests(self) -> None: + base = ( + "on: push\n" + "jobs:\n" + " build:\n" + " runs-on: ubuntu-latest\n" + " steps:\n" + " - run: cargo build\n" + " test:\n" + " runs-on: ${{ matrix.os }}\n" + " strategy:\n" + " matrix:\n" + " os: [ubuntu-latest, windows-latest]\n" + " steps:\n" + " - name: Prepare\n" + " run: cargo fetch\n" + " - uses: ./.github/actions/validate-nvx\n" + " - run: cargo test --locked # every crate\n" + ) + uses = " - uses: ./.github/actions/validate-nvx\n" + for head, expected in ( + (base.replace("cargo build", "cargo build --release"), []), + (base.replace("cargo fetch", "cargo fetch --locked"), []), + (base.replace("# every crate", "# all crates"), []), + (base.replace(uses, f" # {uses.strip()}\n"), ["removed", "removed"]), + ( + base.replace( + " - run: cargo test", + " - if: false\n run: cargo test", + ), + ["added a line that may skip", "modified"], + ), + (base.replace(", windows-latest", ""), ["modified"]), + ( + base.replace( + " strategy:", " continue-on-error: true\n strategy:" + ), + ["added a line that may skip"], + ), + ( + base + " continue-on-error: true\n", + ["added a line that may skip"], + ), + (base.replace(" test:\n", " tests:\n"), ["removed"]), + (base.replace(uses, uses + " - run: cargo test --doc\n"), []), + ): + with self.subTest(head=head): + found = changes.token_changes( + "ci.yml", base, head + ) + changes.job_changes("ci.yml", base, head) + self.assertEqual(sorted(change.action for change in found), expected) + nested = base.replace( + " - run: cargo test --locked # every crate\n", + " - run: cargo test --locked\n env:\n if: 'false'\n", + ) + dedented = nested.replace(" if: 'false'", " if: 'false'") + self.assertEqual( + sorted( + change.action + for change in changes.job_changes("ci.yml", nested, dedented) + ), + ["added a line that may skip", "modified"], + ) + jobs = changes.workflow_jobs(base) + self.assertEqual( + { + name: ( + job.line, + [text for _, text in job.keys], + [[text for _, text in step] for step in job.steps], + job.runs_tests(), + ) + for name, job in jobs.items() + }, + { + "build": ( + 3, + [" runs-on: ubuntu-latest"], + [[" - run: cargo build"]], + False, + ), + "test": ( + 7, + [ + " runs-on: ${{ matrix.os }}", + " strategy:", + " matrix:", + " os: [ubuntu-latest, windows-latest]", + ], + [ + [" - name: Prepare", " run: cargo fetch"], + [" - uses: ./.github/actions/validate-nvx"], + [" - run: cargo test --locked"], + ], + True, + ), + }, + ) + self.assertEqual(changes.job_changes("ci.yml", None, base), []) + + def test_scenarios_that_leave_the_shards(self) -> None: + def shards(*scenarios: str) -> str: + return json.dumps( + {"backends": {"kvm": {"shards": [{"scenarios": list(scenarios)}]}}} + ) + + self.assertEqual( + changes.shard_changes(shards("a", "b"), shards("b", "a", "c")), [] + ) + self.assertEqual( + [ + change.what + for change in changes.shard_changes(shards("a", "b"), shards("a")) + ], + ["`b` on kvm"], + ) + self.assertEqual( + [change.action for change in changes.shard_changes(shards("a"), "{")], + ["modified"], + ) + + def test_allowlist_entries_that_allow_more(self) -> None: + def allowlist( + baseline: Mapping[str, tuple[int, int]], seams: Mapping[str, int] + ) -> str: + return layout.render_allowlist( + layout.Allowlist(dict(baseline), dict(seams)) + ) + + base = allowlist({"a.rs": (2, 3), "b.rs": (1, 1)}, {"c.rs": 2}) + lowered = allowlist({"a.rs": (2, 1)}, {"b.rs": 1, "c.rs": 1}) + self.assertEqual(changes.allowlist_changes(base, lowered), []) + raised = allowlist({"a.rs": (2, 4), "b.rs": (1, 1)}, {"c.rs": 2, "d.rs": 1}) + self.assertEqual( + [change.what for change in changes.allowlist_changes(base, raised)], + ["`a.rs`", "`d.rs`"], + ) + self.assertEqual( + [change.action for change in changes.allowlist_changes(base, "[]")], + ["modified"], + ) + self.assertEqual(changes.allowlist_changes(None, raised), []) + # Fewer hooks do not pay for more tests, and a file of the seams that + # moves to the baseline may hold test code again. + traded = allowlist({"a.rs": (1, 4), "b.rs": (1, 1)}, {"c.rs": 2}) + self.assertEqual( + [change.what for change in changes.allowlist_changes(base, traded)], + ["`a.rs`"], + ) + back = allowlist({"a.rs": (2, 3), "b.rs": (1, 1), "c.rs": (1, 0)}, {}) + self.assertEqual( + [change.detail for change in changes.allowlist_changes(base, back)], + ["moves from the seams to the baseline, which allows tests"], + ) + + +class ClassifyTests(unittest.TestCase): + def test_paths_and_their_categories(self) -> None: + crates = ("nvx-rs", "tests/runtime-adapter") + for path, expected in ( + ("scripts/test-layout-allowlist.json", (changes.TEST_POLICY, "allowlist")), + ("scripts/check_test_layout.py", (changes.TEST_POLICY, "file")), + (".github/actions/validate-nvx/action.yml", (changes.CI_WIRING, "lines")), + (".github/workflows/ci.yml", (changes.CI_WIRING, "tokens")), + ("scripts/nvx_tools/microvm_shards.json", (changes.CI_WIRING, "shards")), + ("nvx-rs/src/testing/fake_openvmm.rs", (changes.TEST_DOUBLES, "rust")), + ("nvx-rs/src/bin/nvx-fake-openvmm.rs", (changes.TEST_DOUBLES, "rust")), + ( + "scripts/nvx_tools/microvm_test_scripts/a.sh", + (changes.END_TO_END, "file"), + ), + ("guest/common/nvx-sandbox-smoke", (changes.END_TO_END, "file")), + ("scripts/nvx_tools/microvm_tests.py", (changes.END_TO_END, "python")), + ("scripts/test_nvx_tools.py", (changes.TEST_CODE, "python")), + ("scripts/powershell_testing.py", (changes.TEST_CODE, "python")), + ("tests/runtime-adapter/scenarios.txt", (changes.TEST_DATA, "file")), + ("nvx-rs/tests/fixtures/store/runtime.json", (changes.TEST_DATA, "file")), + ("nvx-rs/tests/vectors/openvmm/a.rs", (changes.TEST_DATA, "file")), + ("nvx-rs/tests/cli.rs", (changes.TEST_CODE, "rust")), + ("nvx-rs/src/a/tests.rs", (changes.TEST_CODE, "rust")), + ("tests/runtime-adapter/tests/runtime.rs", (changes.TEST_CODE, "rust")), + ("nvx-rs/src/a.rs", (changes.TEST_CODE, "rust-production")), + ("nvx-rs/tests/vectors/README.md", None), + ("nvx-rs/README.md", None), + ("scripts/nvx_tools/benchmark.py", None), + ("doc/ci.md", None), + ): + with self.subTest(path=path): + self.assertEqual(changes.classify(path, crates), expected) + + +class Repository: + """A Git repository in a temporary directory.""" + + def __init__(self, root: Path) -> None: + self.root = root + self.git("init", "-q", "-b", "dev") + + def git(self, *arguments: str) -> str: + return subprocess.run( + [ + "git", + "-c", + "core.autocrlf=false", + "-c", + "commit.gpgsign=false", + "-c", + "user.name=Test", + "-c", + "user.email=test@example.com", + *arguments, + ], + cwd=self.root, + check=True, + capture_output=True, + text=True, + ).stdout + + def commit(self, files: Mapping[str, str | None]) -> str: + for path, text in files.items(): + target = self.root / path + if text is None: + target.unlink() + continue + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(text, encoding="utf-8", newline="\n") + self.git("add", "-A") + self.git("commit", "-q", "--allow-empty", "-m", "change") + return self.git("rev-parse", "HEAD").strip() + + +class CommitTests(unittest.TestCase): + def setUp(self) -> None: + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.repository = Repository(Path(directory.name)) + self.base = self.repository.commit( + { + "demo/Cargo.toml": CARGO, + "demo/src/lib.rs": "pub mod a;\n", + "demo/src/a.rs": INLINE, + "demo/tests/fixtures/expected.json": '{"value": 1}\n', + "demo/tests/fixtures/old.json": "{}\n", + "scripts/nvx_tools/microvm_test_scripts/boot.sh": "echo boot\n", + "scripts/test_tool.py": PythonEntryTests.BASE, + "notes/guide.md": "# Guide\n", + } + ) + + def compare(self, head: str) -> changes.Comparison: + return changes.compare_commits(self.repository.root, self.base, head) + + def test_a_declaration_that_changes_where_its_module_compiles_counts(self) -> None: + unchanged = self.repository.commit( + {"demo/src/lib.rs": "pub mod a;\npub mod b;\n"} + ) + self.assertEqual(self.compare(unchanged).changes, []) + head = self.repository.commit( + {"demo/src/lib.rs": "#[cfg(windows)]\npub mod a;\n"} + ) + self.assertIn( + ( + "test", + "it now compiles where `test` and `windows` hold, not where `test` did", + ), + [(change.kind, change.detail) for change in self.compare(head).flagged], + ) + head = self.repository.commit( + {"demo/src/lib.rs": "#[cfg_attr(unix, cfg(any()))]\npub mod a;\n"} + ) + self.assertIn( + ( + "test", + "it now compiles where `any ( not ( unix ) , any ( ) )` and `test` " + "hold, not where `test` did", + ), + [(change.kind, change.detail) for change in self.compare(head).flagged], + ) + # A module that no longer compiles takes its tests with it. + head = self.repository.commit({"demo/src/lib.rs": "pub mod b;\n"}) + self.assertIn( + ("removed", "test", "`fn works` in `a::tests`"), + [ + (change.action, change.kind, change.what) + for change in self.compare(head).flagged + ], + ) + + def test_every_target_that_compiles_a_module_counts(self) -> None: + both = self.repository.commit( + { + "demo/src/lib.rs": "#[cfg(windows)]\nmod a;\n", + "demo/src/main.rs": "#[cfg(unix)]\nmod a;\n\nfn main() {}\n", + } + ) + # The binary compiles the module still, but no longer where the library + # did. + binary = self.repository.commit({"demo/src/lib.rs": ""}) + comparison = changes.compare_commits(self.repository.root, both, binary) + self.assertIn( + ( + "test", + "it now compiles where `test` and `unix` hold, not where " + "(`test` and `unix`) or (`test` and `windows`) did", + ), + [(change.kind, change.detail) for change in comparison.flagged], + ) + + def test_a_declaration_in_another_directory_counts(self) -> None: + cases = "#[test]\nfn case() {}\n" + reached = self.repository.commit( + { + "demo/tests/cli.rs": "mod suite;\n", + "demo/tests/suite/mod.rs": '#[path = "../shared/cases.rs"]\nmod cases;\n', + "demo/tests/shared/cases.rs": cases, + } + ) + # Only the declarer changes, in a directory apart from its module's. + guarded = self.repository.commit( + { + "demo/tests/suite/mod.rs": '#[cfg(windows)]\n#[path = "../shared/cases.rs"]\n' + "mod cases;\n" + } + ) + comparison = changes.compare_commits(self.repository.root, reached, guarded) + self.assertIn( + ("modified", "test", "it now compiles only where `windows` hold"), + [ + (change.action, change.kind, change.detail) + for change in comparison.flagged + ], + ) + dropped = self.repository.commit({"demo/tests/suite/mod.rs": ""}) + comparison = changes.compare_commits(self.repository.root, reached, dropped) + self.assertIn( + ("removed", "test", "demo/tests/shared/cases.rs"), + [ + (change.action, change.kind, cast(changes.Location, change.base).path) + for change in comparison.flagged + ], + ) + + def test_a_manifest_that_stops_testing_a_target_removes_its_tests(self) -> None: + unchanged = self.repository.commit( + {"demo/Cargo.toml": CARGO + '\n[dependencies]\nlibc = "0.2"\n'} + ) + self.assertEqual(self.compare(unchanged).changes, []) + head = self.repository.commit( + {"demo/Cargo.toml": CARGO + "\n[lib]\ntest = false\n"} + ) + self.assertIn( + ("removed", "test", "`fn works` in `a::tests`"), + [ + (change.action, change.kind, change.what) + for change in self.compare(head).flagged + ], + ) + restored = self.repository.commit({"demo/Cargo.toml": CARGO}) + comparison = changes.compare_commits(self.repository.root, head, restored) + self.assertEqual(comparison.flagged, []) + self.assertIn( + ("added", "test"), + [(change.action, change.kind) for change in comparison.changes], + ) + # A library that the manifest roots elsewhere no longer compiles the + # modules of src/lib.rs, which stays. + moved = self.repository.commit( + { + "demo/Cargo.toml": CARGO + '\n[lib]\npath = "src/core.rs"\n', + "demo/src/core.rs": "", + } + ) + self.assertIn( + ("removed", "test", "`fn works` in `a::tests`"), + [ + (change.action, change.kind, change.what) + for change in changes.compare_commits( + self.repository.root, restored, moved + ).flagged + ], + ) + + def test_moves_and_new_tests_flag_nothing(self) -> None: + head = self.repository.commit( + { + "demo/src/a.rs": DECLARED, + "demo/src/a/tests.rs": MOVED + "\n#[test]\nfn more() {}\n", + "demo/tests/fixtures/old.json": None, + "demo/tests/fixtures/renamed.json": "{}\n", + "notes/guide.md": "# Guide\n\nMore.\n", + } + ) + comparison = self.compare(head) + self.assertEqual(comparison.flagged, []) + self.assertEqual(comparison.count("relocated"), 5) + self.assertEqual(comparison.count("added"), 1) + self.assertIn("changes no existing test", changes.render(comparison, None)) + + def test_changes_to_existing_tests_flag(self) -> None: + head = self.repository.commit( + { + "demo/src/a.rs": INLINE.replace("EXPECTED);", "EXPECTED + 0);"), + "demo/tests/fixtures/expected.json": '{"value": 2}\n', + "scripts/nvx_tools/microvm_test_scripts/boot.sh": None, + "scripts/test_tool.py": PythonEntryTests.BASE.replace( + "(1, 1)", "(1, 2)" + ), + } + ) + comparison = self.compare(head) + self.assertEqual( + flagged(comparison.flagged), + [ + (changes.END_TO_END, "removed", "file"), + (changes.TEST_CODE, "modified", "test"), + (changes.TEST_CODE, "modified", "test"), + (changes.TEST_DATA, "modified", "file"), + ], + ) + report = changes.render(comparison, "owner/repo") + self.assertTrue(report.startswith(changes.COMMENT_MARKER)) + self.assertIn("modifies or removes 4 existing test asset(s)", report) + self.assertIn( + f"(https://github.com/owner/repo/blob/{comparison.head}/demo/src/a.rs#L", + report, + ) + self.assertIn("#### Test data", report) + + def test_the_merge_base_counts(self) -> None: + topic = self.repository.commit({"demo/tests/fixtures/new.json": "{}\n"}) + self.repository.git("checkout", "-q", "-b", "other", self.base) + moved_on = self.repository.commit( + {"demo/tests/fixtures/expected.json": '{"value": 3}\n'} + ) + comparison = changes.compare_commits(self.repository.root, moved_on, topic) + self.assertEqual(comparison.base, self.base) + self.assertEqual(comparison.flagged, []) + + def test_modules_that_only_tests_compile_are_test_code(self) -> None: + support = "pub(crate) fn fixture() -> u8 {\n 1\n}\n" + base = self.repository.commit( + { + "demo/src/lib.rs": "pub mod a;\n#[cfg(test)]\nmod support;\n", + "demo/src/support.rs": support, + "demo/src/a.rs": "mod inner {\n #[cfg(test)]\n mod model;\n}\n", + "demo/src/a/inner/model.rs": support, + } + ) + head = self.repository.commit( + { + "demo/src/support.rs": support.replace("1", "2"), + "demo/src/a/inner/model.rs": support.replace("1", "3"), + } + ) + comparison = changes.compare_commits(self.repository.root, base, head) + self.assertEqual( + sorted( + (change.action, change.kind, cast(changes.Location, change.base).path) + for change in comparison.flagged + ), + [ + ("modified", "helper", "demo/src/a/inner/model.rs"), + ("modified", "helper", "demo/src/support.rs"), + ], + ) + # So does a module that a root that the manifest names declares. + rooted = self.repository.commit( + { + "demo/Cargo.toml": CARGO + '\n[lib]\npath = "src/core.rs"\n', + "demo/src/core.rs": "pub mod a;\n#[cfg(test)]\nmod helpers;\n", + "demo/src/helpers.rs": support, + } + ) + edited = self.repository.commit( + {"demo/src/helpers.rs": support.replace("1", "4")} + ) + comparison = changes.compare_commits(self.repository.root, rooted, edited) + self.assertEqual( + [ + (change.action, change.kind, cast(changes.Location, change.base).path) + for change in comparison.flagged + ], + [("modified", "helper", "demo/src/helpers.rs")], + ) + + def test_main_reports_and_writes_outputs(self) -> None: + head = self.repository.commit( + {"demo/tests/fixtures/expected.json": '{"value": 2}\n'} + ) + root = self.repository.root + report, output = root / "report.md", root / "output.txt" + printed = io.StringIO() + with contextlib.redirect_stdout(printed): + status = changes.main( + [ + "--root", + str(root), + "--base", + self.base, + "--head", + head, + "--report", + str(report), + "--github-output", + str(output), + ] + ) + self.assertEqual(status, 0) + self.assertIn( + "test data: modified file demo/tests/fixtures/expected.json", + printed.getvalue(), + ) + self.assertEqual(output.read_text(encoding="utf-8"), "flagged=true\ncount=1\n") + self.assertTrue( + report.read_text(encoding="utf-8").startswith(changes.COMMENT_MARKER) + ) + with contextlib.redirect_stderr(io.StringIO()): + self.assertEqual( + changes.main(["--root", str(root), "--base", "missing"]), 2 + ) + with self.assertRaises(SystemExit): + changes.main(["--root", str(root), "--publish", "1"]) + + +class RenderTests(unittest.TestCase): + def test_comments_fit_and_mention_no_one(self) -> None: + comparison = changes.Comparison("a" * 40, "b" * 40) + comparison.changes = [ + changes.Change( + changes.TEST_CODE, + "removed", + "test", + f"`fn test_{index}` of @someone | x", + changes.Location("src/a.rs", index + 1), + ) + for index in range(3000) + ] + body = changes.comment_body(comparison, "owner/repo") + self.assertLessEqual(len(body), changes.COMMENT_LIMIT) + self.assertIn("more, which the job's summary lists", body) + self.assertNotIn("@someone", body) + self.assertIn("\\|", body) + + +class FakeGh: + """Answers gh's calls from a script of responses.""" + + def __init__( + self, responses: Mapping[tuple[str, str], tuple[int, str, str]] + ) -> None: + self.responses = responses + self.calls: list[tuple[str, str, str | None]] = [] + + def __call__( + self, arguments: Sequence[str], stdin: str | None + ) -> subprocess.CompletedProcess[str]: + method = arguments[arguments.index("--method") + 1] + endpoint = next( + argument for argument in arguments[2:] if argument.startswith("repos/") + ) + self.calls.append((method, endpoint, stdin)) + status, stdout, stderr = self.responses.get((method, endpoint), (0, "", "")) + return subprocess.CompletedProcess(list(arguments), status, stdout, stderr) + + +class PublishTests(unittest.TestCase): + LABEL = "repos/o/r/labels/tests-modified" + ISSUE_LABELS = "repos/o/r/issues/7/labels" + COMMENTS = "repos/o/r/issues/7/comments" + + def publish( + self, + flagged: bool, + responses: Mapping[tuple[str, str], tuple[int, str, str]], + ) -> list[tuple[str, str, str | None]]: + gh = FakeGh(responses) + changes.GitHub("o/r", gh).publish(7, "body", flagged) + return gh.calls + + def test_a_flagged_pull_request_gets_the_label_and_a_comment(self) -> None: + calls = self.publish( + flagged=True, responses={("GET", self.LABEL): (1, "", "HTTP 404")} + ) + self.assertEqual( + [(method, endpoint) for method, endpoint, _ in calls], + [ + ("GET", self.LABEL), + ("POST", "repos/o/r/labels"), + ("POST", self.ISSUE_LABELS), + ("GET", self.COMMENTS), + ("POST", self.COMMENTS), + ], + ) + self.assertEqual( + json.loads(cast(str, calls[2][2])), {"labels": ["tests-modified"]} + ) + self.assertEqual(json.loads(cast(str, calls[4][2])), {"body": "body"}) + + def test_the_comment_is_updated_and_the_label_removed(self) -> None: + calls = self.publish( + flagged=False, + responses={ + ("DELETE", f"{self.ISSUE_LABELS}/tests-modified"): (1, "", "HTTP 404"), + ("GET", self.COMMENTS): (0, "11\n12\n", ""), + }, + ) + self.assertEqual( + [(method, endpoint) for method, endpoint, _ in calls], + [ + ("DELETE", f"{self.ISSUE_LABELS}/tests-modified"), + ("GET", self.COMMENTS), + ("PATCH", "repos/o/r/issues/comments/11"), + ], + ) + self.assertEqual( + [ + endpoint + for method, endpoint, _ in self.publish(False, {}) + if method != "GET" + ], + [f"{self.ISSUE_LABELS}/tests-modified"], + ) + + def test_failures_raise(self) -> None: + with self.assertRaises(changes.FlagError): + self.publish(True, {("POST", self.ISSUE_LABELS): (1, "", "HTTP 403")}) + + def test_a_label_that_another_run_created_first_is_no_failure(self) -> None: + calls = self.publish( + True, + { + ("GET", self.LABEL): (1, "", "HTTP 404"), + ("POST", "repos/o/r/labels"): (1, "", "HTTP 422: already_exists"), + }, + ) + self.assertIn(("POST", self.ISSUE_LABELS), [call[:2] for call in calls]) + + +class WorkflowTests(unittest.TestCase): + def setUp(self) -> None: + self.workflow = cast( + dict[object, object], yaml.safe_load(WORKFLOW.read_text(encoding="utf-8")) + ) + + def test_it_flags_pull_requests_without_running_their_code(self) -> None: + # YAML 1.1 reads the `on` key as True. + triggers = cast(dict[str, object], self.workflow[True]) + self.assertEqual(list(triggers), ["pull_request_target"]) + self.assertEqual(self.workflow["permissions"], {}) + jobs = cast(dict[str, dict[str, object]], self.workflow["jobs"]) + [job] = jobs.values() + self.assertEqual( + job["permissions"], {"contents": "read", "pull-requests": "write"} + ) + steps = cast(list[dict[str, object]], job["steps"]) + checkouts = [ + step + for step in steps + if str(step.get("uses", "")).startswith("actions/checkout@") + ] + self.assertEqual(len(checkouts), 1) + options = cast(dict[str, object], checkouts[0]["with"]) + # The default branch's script runs, never the pull request's. + self.assertEqual( + options["ref"], "${{ github.event.repository.default_branch }}" + ) + self.assertNotIn("repository", options) + self.assertIs(options["persist-credentials"], False) + # A skipped job, as an edit of the title skips, holds no concurrency + # group, so it cancels no comparison. + self.assertNotIn("concurrency", self.workflow) + self.assertEqual( + job["concurrency"], + { + "group": "test-changes-${{ github.event.pull_request.number }}", + "cancel-in-progress": True, + }, + ) + scripts = [str(step["run"]) for step in steps if "run" in step] + self.assertTrue(scripts) + # The run flags the head of its event only while it is the pull + # request's head. + self.assertIn( + '"$(git rev-parse refs/remotes/pull/head)" == "${HEAD_SHA}"', scripts[0] + ) + for script in scripts: + self.assertNotIn("${{", script) + for forbidden in ("pip ", "cargo ", "npm ", "make ", "bash ./", "sh ./"): + self.assertNotIn(forbidden, script) + self.assertIn("scripts/check_test_changes.py", "\n".join(scripts)) + self.assertIn("--publish", "\n".join(scripts)) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_test_layout.py b/scripts/test_test_layout.py new file mode 100644 index 000000000..f00753a64 --- /dev/null +++ b/scripts/test_test_layout.py @@ -0,0 +1,879 @@ +"""Tests of check_test_layout.py, the check of where tests live.""" + +from __future__ import annotations + +import contextlib +import io +import os +import subprocess +import sys +import tempfile +import unittest +from collections.abc import Mapping +from pathlib import Path +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import check_test_layout as layout # noqa: E402 + +CARGO = '[package]\nname = "demo"\nversion = "0.1.0"\n' + + +class FakeTree: + """The tracked files of a tree, in memory.""" + + def __init__(self, files: Mapping[str, str]) -> None: + self._files = {path: text.encode("utf-8") for path, text in files.items()} + + @property + def files(self) -> frozenset[str]: + return frozenset(self._files) + + def read(self, path: str) -> bytes: + return self._files[path] + + +def crate(files: Mapping[str, str], allowlist: str | None = None) -> FakeTree: + """A tree with one crate, demo/, that holds files, by their paths in it.""" + tree = {"demo/Cargo.toml": CARGO} + tree.update({f"demo/{path}": text for path, text in files.items()}) + if allowlist is not None: + tree[layout.ALLOWLIST] = allowlist + return FakeTree(tree) + + +def allowlist( + baseline: Mapping[str, tuple[int, int]] | None = None, + seams: Mapping[str, int] | None = None, +) -> str: + return layout.render_allowlist( + layout.Allowlist(dict(baseline or {}), dict(seams or {})) + ) + + +def rules(tree: layout.Snapshot) -> list[tuple[str, int, str]]: + return [ + (problem.path, problem.line, problem.rule) + for problem in layout.check_snapshot(tree).problems + ] + + +LIBRARY = "pub mod a;\n" +TESTED = "pub fn f() -> u8 {\n 1\n}\n\n#[cfg(test)]\nmod tests;\n" +TESTS = "use super::*;\n\n#[test]\nfn works() {\n assert_eq!(f(), 1);\n}\n" + + +class TokenizeTests(unittest.TestCase): + def texts(self, source: str) -> list[str]: + return [token.text for token in layout.tokenize_rust(source)] + + def test_comments_are_no_tokens(self) -> None: + source = "a // b #[cfg(test)]\n/* c /* d */ e */ f\n/// g\n//! h\n/** i */ j" + self.assertEqual(self.texts(source), ["a", "f", "j"]) + + def test_literals_hide_what_looks_like_code(self) -> None: + source = ( + 'x("// #[cfg(test)] /* \\" ", r#"a " b"#, br##"#"##, b"\\\\", c"z",' + " '\"', '{', '\\'', '\\\\', '\\u{1F600}', b'\\'')" + ) + tokens = layout.tokenize_rust(source) + self.assertEqual( + [token.text for token in tokens if token.kind == layout.LITERAL], + [ + '"// #[cfg(test)] /* \\" "', + 'r#"a " b"#', + 'br##"#"##', + 'b"\\\\"', + 'c"z"', + "'\"'", + "'{'", + "'\\''", + "'\\\\'", + "'\\u{1F600}'", + "b'\\''", + ], + ) + self.assertEqual(layout.match_brackets(tokens)[1], len(tokens) - 1) + + def test_lifetimes_and_labels_are_no_char_literals(self) -> None: + source = ( + "fn f<'a>(x: &'a str) -> &'static str { 'outer: loop { break 'outer; } }" + ) + tokens = layout.tokenize_rust(source) + self.assertEqual( + [token.text for token in tokens if token.kind == layout.LIFETIME], + ["'a", "'a", "'static", "'outer", "'outer"], + ) + self.assertEqual( + [token.text for token in layout.tokenize_rust("['a', 'b']")], + ["[", "'a'", ",", "'b'", "]"], + ) + + def test_raw_identifiers_and_numbers(self) -> None: + self.assertEqual(self.texts("r#type 0..10 1.5e3f64 x.0"), [ + "r#type", "0", ".", ".", "10", "1.5e3f64", "x", ".", "0", + ]) # fmt: skip + self.assertEqual(layout.unraw("r#type"), "type") + + def test_tokens_know_their_lines(self) -> None: + tokens = layout.tokenize_rust('a\n"b\nc"\n\nd') + self.assertEqual([token.line for token in tokens], [1, 2, 5]) + + def test_unterminated_literals_and_comments_fail(self) -> None: + for source, line in (('a\n"b', 2), ("/* a\n", 1), ('r#"a"', 1)): + with self.subTest(source=source): + with self.assertRaises(layout.RustSyntaxError) as raised: + layout.tokenize_rust(source) + self.assertEqual(raised.exception.line, line) + + def test_unbalanced_brackets_fail(self) -> None: + for source in ("(]", "{", "}"): + with self.subTest(source=source): + with self.assertRaises(layout.RustSyntaxError): + layout.match_brackets(layout.tokenize_rust(source)) + + +class CfgTests(unittest.TestCase): + def cfg(self, source: str) -> layout.Cfg: + tokens = layout.tokenize_rust(source) + partner = layout.match_brackets(tokens) + return layout.parse_cfg(tokens, partner, 0, len(tokens)) + + def test_requires_forbids_and_mentions_test(self) -> None: + for source, requires, forbids, mentions in ( + ("test", True, False, True), + ("not(test)", False, True, True), + ("all(test, unix)", True, False, True), + ("all(unix, not(test))", False, True, True), + ('any(test, feature = "cli")', False, False, True), + ("any(test, all(test, unix))", True, False, True), + ("not(not(test))", True, False, True), + ("any()", False, False, False), + ('feature = "test"', False, False, False), + ("unix", False, False, False), + ("$test", False, False, False), + ("all(test, $rest)", True, False, True), + ): + with self.subTest(source=source): + cfg = self.cfg(source) + self.assertEqual(layout.requires_test(cfg), requires) + self.assertEqual(layout.forbids_test(cfg), forbids) + self.assertEqual(layout.mentions_test(cfg), mentions) + + def test_attributes(self) -> None: + source = ( + "#[cfg(test)] #[cfg_attr(test, derive(Debug))] #[cfg_attr(unix, cfg(test))] " + '#[cfg_attr(unix, path = "x.rs")] #[tokio::test] #![allow(dead_code)]' + ) + tokens = layout.tokenize_rust(source) + partner = layout.match_brackets(tokens) + attributes = [ + attribute + for index in range(len(tokens)) + if (attribute := layout.attribute_at(tokens, partner, index)) is not None + ] + self.assertEqual( + [(attribute.path, attribute.inner) for attribute in attributes], + [ + ("cfg", False), + ("cfg_attr", False), + ("cfg_attr", False), + ("cfg_attr", False), + ("tokio::test", False), + ("allow", True), + ], + ) + self.assertEqual( + [ + layout.attribute_mentions_test(tokens, partner, attribute) + for attribute in attributes + ], + [True, True, True, False, False, False], + ) + self.assertEqual( + [ + layout.attribute_requires_test(tokens, partner, attribute) + for attribute in attributes + ], + [True, False, False, False, False, False], + ) + self.assertIn("path", layout.attribute_paths(tokens, partner, attributes[3])) + self.assertTrue(layout.is_test_attribute("tokio::test")) + self.assertTrue(layout.is_test_attribute("should_panic")) + self.assertFalse(layout.is_test_function_attribute("ignore")) + + +class ItemTests(unittest.TestCase): + def test_items_end_where_rust_ends_them(self) -> None: + source = """ +#![allow(unused)] +use std::{collections::HashMap, fmt}; +extern crate alloc as heap; +pub(crate) const LIMIT: Limit = Limit { value: { 3 } }; +const _: () = assert!(true); +static mut COUNT: u8 = 0; +pub const fn twice(x: u8) -> u8 { x * 2 } +pub unsafe extern "C" fn raw() {} +unsafe extern "C" { fn ffi(); static OUTSIDE: u8; } +struct Unit; +struct Pair(u8, u8); +struct Named where T: Copy { field: T } +pub enum Kind { A = { 1 }, B } +union Bits { a: u8 } +trait Shape: Send { const SIDES: u8; fn area(&self) -> u8; type Unit; } +unsafe impl Send for Named where T: Copy {} +impl Pair { pub fn sum(&self) -> u8 { self.0 + self.1 } #[cfg(test)] fn probe() {} } +type Alias = Pair; +macro_rules! square { ($x:expr) => { $x * $x }; } +macro_rules! cube ( ($x:expr) => { $x * $x * $x } ); +thread_local! { static SEEN: u8 = const { 0 }; } +std::println!("{}", 1); +mod inline { fn inner() {} } +mod declared; +""" + parsed = layout.parse_rust(source) + self.assertEqual( + [attribute.path for attribute in parsed.inner_attributes], ["allow"] + ) + self.assertEqual( + [(item.kind, item.name) for item in parsed.items], + [ + ("use", ""), + ("extern crate", "alloc"), + ("const", "LIMIT"), + ("const", "_"), + ("static", "COUNT"), + ("fn", "twice"), + ("fn", "raw"), + ("extern", ""), + ("struct", "Unit"), + ("struct", "Pair"), + ("struct", "Named"), + ("enum", "Kind"), + ("union", "Bits"), + ("trait", "Shape"), + ("impl", "impl < T > Send for Named < T > where T : Copy"), + ("impl", "impl Pair"), + ("type", "Alias"), + ("macro_rules", "square"), + ("macro_rules", "cube"), + ("macro", "thread_local"), + ("macro", "std::println"), + ("mod", "inline"), + ("mod", "declared"), + ], + ) + children = {item.name: item.children for item in parsed.items} + self.assertEqual( + [(child.kind, child.name) for child in children["impl Pair"]], + [("fn", "sum"), ("fn", "probe")], + ) + self.assertEqual( + [(child.kind, child.name) for child in children["Shape"]], + [("const", "SIDES"), ("fn", "area"), ("type", "Unit")], + ) + self.assertEqual( + [(child.kind, child.name) for child in children[""]], + [("fn", "ffi"), ("static", "OUTSIDE")], + ) + self.assertEqual([child.name for child in children["inline"]], ["inner"]) + self.assertIsNone(parsed.items[-1].body) + + def test_braces_in_generic_arguments_open_no_body(self) -> None: + parsed = layout.parse_rust( + "impl S<{ 1 }> {\n fn a() {}\n}\n" + "fn g u8>(f: F) -> Option where F: Copy {\n None\n}\n" + "fn compare() -> bool { 1 < 2 }\n" + "fn after() {}\n" + ) + self.assertEqual( + [(item.kind, item.name) for item in parsed.items], + [ + ("impl", "impl S < { 1 } >"), + ("fn", "g"), + ("fn", "compare"), + ("fn", "after"), + ], + ) + self.assertEqual([child.name for child in parsed.items[0].children], ["a"]) + + +class ModulePathTests(unittest.TestCase): + def test_roles_and_module_files(self) -> None: + for path, role in ( + ("demo/src/a.rs", "production"), + ("demo/src/a/tests.rs", "test"), + ("demo/src/a/tests/b.rs", "test"), + ("demo/src/tests/mod.rs", "test"), + ("demo/tests/cli.rs", "test"), + ("demo/build.rs", ""), + ("demo/src/a.txt", ""), + ): + with self.subTest(path=path): + self.assertEqual(layout.rust_role(path, "demo"), role) + self.assertEqual( + layout.module_files("demo", "a/b.rs", ("inner",), "tests"), + ("demo/src/a/b/inner/tests.rs", "demo/src/a/b/inner/tests/mod.rs"), + ) + self.assertEqual( + layout.module_files("demo", "a/mod.rs", (), "tests")[0], + "demo/src/a/tests.rs", + ) + self.assertEqual( + layout.module_files("demo", "lib.rs", (), "tests")[0], "demo/src/tests.rs" + ) + self.assertEqual( + layout.module_files("demo", "bin/tool.rs", (), "x")[0], + "demo/src/bin/x.rs", + ) + self.assertEqual(layout.module_path("a/b/mod.rs"), ("a", "b")) + self.assertEqual(layout.module_path("lib.rs"), ()) + self.assertEqual(layout.module_path("bin/tool.rs"), ("bin:tool",)) + self.assertEqual(layout.crate_of("x/demo/src/a.rs", ("", "x/demo")), "x/demo") + + +class RustRuleTests(unittest.TestCase): + def test_tests_in_their_own_files_pass(self) -> None: + tree = crate( + {"src/lib.rs": LIBRARY, "src/a.rs": TESTED, "src/a/tests.rs": TESTS} + ) + self.assertEqual(rules(tree), []) + + def test_test_code_in_production_files_fails(self) -> None: + for source, rule in ( + ( + "#[cfg(test)]\nmod tests {\n #[test]\n fn t() {}\n}\n", + "inline-test-module", + ), + ("#[cfg(all(test, unix))]\nmod checks {}\n", "inline-test-module"), + ("mod checks {\n #![cfg(test)]\n}\n", "inline-test-module"), + ("#[cfg(test)]\nmod support;\n", "test-module-name"), + ("#[cfg(test)]\nfn helper() {}\n", "test-only-item"), + ("#[cfg_attr(all(), cfg(test))]\nfn helper() {}\n", "test-only-item"), + ( + "struct S;\nimpl S<{ 1 }> {\n #[cfg(test)]\n" + " fn helper() {}\n}\n", + "test-only-item", + ), + ("#[cfg(test)]\nuse std::fmt;\n", "test-only-item"), + ( + "struct S;\nimpl S {\n #[cfg(test)]\n fn probe() {}\n}\n", + "test-only-item", + ), + ("#[test]\nfn stray() {}\n", "test-harness"), + ("#[cfg_attr(test, test)]\nfn applied() {}\n", "test-harness"), + ("proptest! {\n #[test]\n fn p(x in 0..1u8) {}\n}\n", "test-harness"), + ("#![cfg(test)]\nfn f() {}\n", "test-only-file"), + ): + with self.subTest(source=source): + tree = crate( + { + "src/lib.rs": LIBRARY, + "src/a.rs": source, + "src/a/support.rs": "", + } + ) + found = rules(tree) + self.assertIn(rule, {found_rule for _, _, found_rule in found}) + self.assertTrue(all(path == "demo/src/a.rs" for path, _, _ in found)) + + def test_tests_modules_require_test_and_their_files(self) -> None: + for files, rule in ( + ({"src/a.rs": "mod tests;\n", "src/a/tests.rs": ""}, "tests-module"), + ({"src/a.rs": "mod tests {}\n"}, "tests-module"), + ({"src/a.rs": "#[cfg(test)]\nmod tests;\n"}, "tests-module"), + ( + { + "src/a.rs": "#[cfg(test)]\nmod tests;\n", + "src/a/tests.rs": "", + "src/a/tests/mod.rs": "", + }, + "tests-module", + ), + ({"src/a.rs": "", "src/a/tests.rs": ""}, "orphan-test-file"), + ( + {"src/a.rs": '#[path = "b.rs"]\nmod b;\n', "src/b.rs": ""}, + "path-attribute", + ), + ): + with self.subTest(files=files): + found = rules(crate({"src/lib.rs": LIBRARY, **files})) + self.assertIn(rule, {found_rule for _, _, found_rule in found}) + + def test_platform_only_tests_modules_need_their_files_too(self) -> None: + tree = crate( + { + "src/lib.rs": LIBRARY, + "src/a.rs": "#[cfg(all(test, windows))]\nmod tests;\n", + } + ) + self.assertEqual(rules(tree), [("demo/src/a.rs", 2, "tests-module")]) + + def test_test_files_declared_in_test_code_belong_to_a_module(self) -> None: + tree = crate( + { + "src/lib.rs": LIBRARY, + "src/a.rs": "#[cfg(test)]\nmod tests;\n", + "src/a/tests.rs": "mod vectors;\nmod inner {\n mod deep;\n}\n", + "src/a/tests/vectors.rs": "", + "src/a/tests/inner/deep.rs": "", + } + ) + self.assertEqual(rules(tree), []) + + def test_test_files_that_no_target_reaches_never_compile(self) -> None: + files = {"src/a.rs": TESTED, "src/a/tests.rs": TESTS} + self.assertEqual(rules(crate({"src/lib.rs": LIBRARY, **files})), []) + self.assertEqual( + rules(crate({"src/lib.rs": "", **files})), + [("demo/src/a/tests.rs", 0, "orphan-test-file")], + ) + binary = { + "src/lib.rs": "", + "src/bin/tool/main.rs": "mod a;\n\nfn main() {}\n", + "src/bin/tool/a.rs": TESTED, + "src/bin/tool/a/tests.rs": TESTS, + } + self.assertEqual(rules(crate(binary)), []) + # Cargo runs the unit tests of no target whose `test` is false. + manifest = ( + CARGO + + '\n[[bin]]\nname = "tool"\npath = "src/bin/tool/main.rs"\ntest = false\n' + ) + self.assertEqual( + rules(crate({**binary, "Cargo.toml": manifest})), + [("demo/src/bin/tool/a/tests.rs", 0, "orphan-test-file")], + ) + + def test_integration_tests_reach_their_modules(self) -> None: + files = { + "src/lib.rs": "", + "tests/cli.rs": '#[path = "cli/harness.rs"]\nmod harness;\nmod common;\n', + "tests/cli/harness.rs": TESTS, + "tests/common/mod.rs": "pub mod gpt;\n", + "tests/common/gpt.rs": "", + "tests/suite/main.rs": 'mod inner {\n #[path = "cases.rs"]\n mod cases;\n}\n', + "tests/suite/inner/cases.rs": TESTS, + # Data, which no test declares. + "tests/fixtures/generate.rs": TESTS, + "tests/vectors/openvmm/a.rs": TESTS, + } + self.assertEqual(rules(crate(files)), []) + unused = {**files, "tests/support/unused.rs": TESTS} + self.assertEqual( + rules(crate(unused)), + [("demo/tests/support/unused.rs", 0, "orphan-test-file")], + ) + manifest = CARGO + '\n[[test]]\nname = "suite"\ntest = false\n' + self.assertEqual( + rules(crate({**files, "Cargo.toml": manifest})), + [ + ("demo/tests/suite/inner/cases.rs", 0, "orphan-test-file"), + ("demo/tests/suite/main.rs", 0, "orphan-test-file"), + ], + ) + + def test_declarations_that_no_test_build_compiles_reach_nothing(self) -> None: + files = { + "src/lib.rs": ( + "#[cfg(any())]\nmod a;\n#[cfg(windows)]\nmod b;\n" + "#[cfg_attr(test, cfg(any()))]\nmod c;\nmod d;\n" + "#[cfg(all(unix, not(all())))]\nmod e {\n mod f;\n}\n" + ), + "src/a.rs": TESTED, + "src/a/tests.rs": TESTS, + "src/b.rs": TESTED, + "src/b/tests.rs": TESTS, + "src/c.rs": TESTED, + "src/c/tests.rs": TESTS, + "src/d.rs": "#![cfg(any())]\n" + TESTED, + "src/d/tests.rs": TESTS, + "src/e/f.rs": TESTED, + "src/e/f/tests.rs": TESTS, + } + # A platform's module compiles where its predicate holds. + self.assertEqual( + rules(crate(files, allowlist(seams={"demo/src/lib.rs": 1}))), + [ + ("demo/src/a/tests.rs", 0, "orphan-test-file"), + ("demo/src/c/tests.rs", 0, "orphan-test-file"), + ("demo/src/d/tests.rs", 0, "orphan-test-file"), + ("demo/src/e/f/tests.rs", 0, "orphan-test-file"), + ], + ) + + def test_manifests_name_the_targets_whose_tests_run(self) -> None: + found = { + "src/lib.rs", + "src/main.rs", + "src/bin/tool.rs", + "src/bin/other/main.rs", + "tests/cli.rs", + "tests/suite/main.rs", + "benches/speed.rs", + "examples/demo.rs", + } + self.assertEqual( + layout.target_roots(CARGO, found), + { + "src/lib.rs": True, + "src/main.rs": True, + "src/bin/tool.rs": True, + "src/bin/other/main.rs": True, + "tests/cli.rs": True, + "tests/suite/main.rs": True, + "benches/speed.rs": False, + "examples/demo.rs": False, + }, + ) + manifest = ( + CARGO + + "autobins = false\nautotests = false\n\n" + + '[lib]\npath = "src/core.rs"\ntest = false\n\n' + + '[[bin]]\nname = "tool"\n\n' + + '[[test]]\nname = "suite"\n\n' + + '[[example]]\nname = "demo"\ntest = true\n' + ) + self.assertEqual( + layout.target_roots(manifest, found), + { + "src/core.rs": False, + "src/bin/tool.rs": True, + "tests/suite/main.rs": True, + "benches/speed.rs": False, + "examples/demo.rs": True, + }, + ) + # A library that the manifest roots elsewhere no longer compiles the + # modules that src/lib.rs declares. + files = { + "src/lib.rs": "mod a;\n", + "src/a.rs": TESTED, + "src/a/tests.rs": TESTS, + "src/core.rs": "mod b;\n", + "src/b.rs": TESTED, + "src/b/tests.rs": TESTS, + } + self.assertEqual( + rules(crate(files)), [("demo/src/b/tests.rs", 0, "orphan-test-file")] + ) + rooted = CARGO + '\n[lib]\npath = "src/core.rs"\n' + self.assertEqual( + rules(crate({**files, "Cargo.toml": rooted})), + [("demo/src/a/tests.rs", 0, "orphan-test-file")], + ) + tests = {"src/lib.rs": "", "tests/auto.rs": TESTS, "tests/named.rs": TESTS} + self.assertEqual(rules(crate(tests)), []) + named = CARGO + 'autotests = false\n\n[[test]]\nname = "named"\n' + self.assertEqual( + rules(crate({**tests, "Cargo.toml": named})), + [("demo/tests/auto.rs", 0, "orphan-test-file")], + ) + # The root of a target whose tests never run still compiles as a module + # that another target declares. + shared = {"src/lib.rs": "", "tests/suite.rs": "mod common;\n"} + unrun = CARGO + '\n[[test]]\nname = "common"\ntest = false\n' + self.assertEqual( + rules(crate({**shared, "tests/common.rs": TESTS, "Cargo.toml": unrun})), [] + ) + # TOML's literal and basic strings both name a file, and a declaration + # of a target that the check cannot read is a problem. + for spelled in ("'src/core.rs'", '"src\\u002fcore.rs"'): + with self.subTest(spelled=spelled): + self.assertEqual( + layout.target_roots( + CARGO + f"\n[lib]\npath = {spelled}\n", {"src/lib.rs"} + ), + {"src/core.rs": True}, + ) + for manifest, line in ( + ('lib = { path = "src/core.rs" }\n' + CARGO, 1), + ('[lib]\npath = """src/core.rs"""\n' + CARGO, 2), + ): + with self.subTest(manifest=manifest): + unread: list[tuple[int, str]] = [] + layout.target_roots(manifest, {"src/lib.rs"}, unread) + self.assertEqual([found for found, _ in unread], [line]) + self.assertIn( + ("demo/Cargo.toml", line, "manifest"), + rules(crate({"src/lib.rs": "", "Cargo.toml": manifest})), + ) + + def test_test_hooks_need_seams(self) -> None: + source = ( + "#[cfg_attr(test, derive(Debug))]\n" + "pub struct S {\n" + " #[cfg(test)]\n" + " seen: u8,\n" + "}\n" + "pub fn f() -> bool {\n" + " #[cfg(test)]\n" + " pause();\n" + " cfg!(test)\n" + "}\n" + '#[cfg(any(test, feature = "cli"))]\n' + "pub fn g() {}\n" + "/// #[cfg(test)] in a doc comment\n" + 'pub const DOC: &str = "#[cfg(test)]";\n' + "#[cfg(test)]\n" + "mod tests;\n" + ) + files = { + "src/lib.rs": LIBRARY, + "src/a.rs": source, + "src/a/tests.rs": "", + } + self.assertEqual(rules(crate(files)), [("demo/src/a.rs", 0, "test-hooks")]) + seams = {"demo/src/a.rs": 5} + self.assertEqual(rules(crate(files, allowlist(seams=seams))), []) + self.assertEqual( + rules(crate(files, allowlist(seams={"demo/src/a.rs": 6}))), + [("demo/src/a.rs", 0, "allowlist")], + ) + self.assertEqual( + rules(crate(files, allowlist(seams={"demo/src/a.rs": 4}))), + [("demo/src/a.rs", 0, "test-hooks")], + ) + analysis = layout.check_snapshot(crate(files)).analyses["demo/src/a.rs"] + self.assertEqual(analysis.hooks, [1, 3, 7, 9, 11]) + + def test_seams_allow_hooks_but_no_test_code(self) -> None: + files = { + "src/lib.rs": LIBRARY, + "src/a.rs": "#[cfg(test)]\nfn helper() {}\n", + } + found = rules(crate(files, allowlist(seams={"demo/src/a.rs": 1}))) + self.assertEqual(found, [("demo/src/a.rs", 2, "test-only-item")]) + + def test_baseline_files_keep_their_test_code_but_no_more(self) -> None: + legacy = ( + "pub fn f() {}\n#[cfg(test)]\nmod tests {\n #[test]\n fn t() {}\n}\n" + ) + files = {"src/lib.rs": LIBRARY, "src/a.rs": legacy} + path = "demo/src/a.rs" + self.assertEqual(rules(crate(files, allowlist({path: (1, 1)}))), []) + grown = legacy.replace("fn t() {}", "fn t() {}\n #[test]\n fn u() {}") + self.assertEqual( + rules(crate({**files, "src/a.rs": grown}, allowlist({path: (1, 1)}))), + [(path, 0, "test-code")], + ) + helper = legacy + "#[cfg(test)]\nfn helper() {}\n" + self.assertEqual( + rules(crate({**files, "src/a.rs": helper}, allowlist({path: (1, 1)}))), + [(path, 0, "test-code")], + ) + self.assertEqual( + rules(crate(files, allowlist({path: (2, 1)}))), [(path, 0, "allowlist")] + ) + moved = { + "src/lib.rs": LIBRARY, + "src/a.rs": "pub fn f() {}\n#[cfg(test)]\nmod tests;\n", + "src/a/tests.rs": "#[test]\nfn t() {}\n", + } + self.assertEqual( + rules(crate(moved, allowlist({path: (1, 1)}))), [(path, 0, "allowlist")] + ) + + def test_unreadable_files_fail(self) -> None: + tree = crate({"src/lib.rs": "fn f() {\n"}) + self.assertEqual(rules(tree), [("demo/src/lib.rs", 1, "rust-syntax")]) + + +class AllowlistTests(unittest.TestCase): + def test_entries_must_name_production_files(self) -> None: + files = {"src/lib.rs": LIBRARY, "src/a.rs": "#[cfg(test)]\nfn f() {}\n"} + both = layout.render_allowlist( + layout.Allowlist({"demo/src/a.rs": (1, 0)}, {"demo/src/a.rs": 1}) + ) + found = rules(crate(files, both)) + self.assertIn((layout.ALLOWLIST, 0, "allowlist"), found) + stale = allowlist(seams={"demo/src/gone.rs": 1}) + self.assertEqual( + rules(crate({"src/lib.rs": LIBRARY}, stale)), + [(layout.ALLOWLIST, 0, "allowlist")], + ) + + def test_the_form_is_checked(self) -> None: + for text in ( + "[]", + "{", + '{"schema": "other", "schema_version": 1, "baseline": {}, "seams": {}}\n', + '{"schema": "nvx-test-layout-allowlist", "schema_version": 1, ' + '"baseline": {}, "seams": {"a": 0}}\n', + allowlist().replace("\n", "\r\n").replace(" ", " "), + ): + with self.subTest(text=text): + errors = layout.parse_allowlist(text)[1] + self.assertNotEqual(errors, []) + self.assertEqual(layout.parse_allowlist(allowlist())[1], []) + + def test_tighten_only_lowers(self) -> None: + analyses = { + "demo/src/lowered.rs": layout.RustAnalysis( + "demo/src/lowered.rs", "demo", True, hooks=[3], tests=1, + legacy=[layout.Problem("demo/src/lowered.rs", 3, "inline-test-module", "")], + ), + "demo/src/moved.rs": layout.RustAnalysis( + "demo/src/moved.rs", "demo", True, hooks=[4, 9] + ), + "demo/src/clean.rs": layout.RustAnalysis("demo/src/clean.rs", "demo", True), + "demo/src/grown.rs": layout.RustAnalysis( + "demo/src/grown.rs", "demo", True, hooks=[1, 2, 3] + ), + "demo/src/fewer.rs": layout.RustAnalysis( + "demo/src/fewer.rs", "demo", True, hooks=[1] + ), + "demo/src/new.rs": layout.RustAnalysis( + "demo/src/new.rs", "demo", True, hooks=[1] + ), + } # fmt: skip + before = layout.Allowlist( + { + "demo/src/lowered.rs": (2, 3), + "demo/src/moved.rs": (5, 4), + "demo/src/clean.rs": (1, 1), + "demo/src/gone.rs": (1, 1), + }, + {"demo/src/grown.rs": 2, "demo/src/fewer.rs": 2}, + ) + after = layout.tighten(before, analyses) + self.assertEqual(after.baseline, {"demo/src/lowered.rs": (1, 1)}) + self.assertEqual( + after.seams, + {"demo/src/moved.rs": 2, "demo/src/grown.rs": 2, "demo/src/fewer.rs": 1}, + ) + + +class PythonRuleTests(unittest.TestCase): + def test_tests_live_where_the_test_jobs_find_them(self) -> None: + for path, allowed in ( + ("scripts/test_a.py", True), + (".github/specula/tests/test_a.py", True), + (".github/skills/demo/scripts/test_a.py", True), + ("scripts/nvx_tools/test_a.py", False), + ("tools/test_a.py", False), + ("scripts/nvx_tools/scenario_tests.py", True), + ("scripts/scenario_tests.py", False), + ): + with self.subTest(path=path): + problems = layout.python_problems(path, "") + self.assertEqual(problems == [], allowed) + + def test_production_modules_import_no_test_code(self) -> None: + for source, allowed in ( + ("import json\nfrom pathlib import Path\n", True), + ("import unittest\n", False), + ("from unittest.mock import patch\n", False), + ("import pytest\n", False), + ("import doctest\n", False), + ("import test_nvx_tools\n", False), + ("from powershell_testing import run_powershell\n", False), + ("from nvx_tools.guest_binaries import test_guest_binaries\n", True), + ("def f(:\n", False), + ): + with self.subTest(source=source): + problems = layout.python_problems("scripts/tool.py", source) + self.assertEqual(problems == [], allowed) + self.assertEqual( + layout.python_problems( + "scripts/powershell_testing.py", "import unittest\n" + ), + [], + ) + self.assertEqual( + layout.python_problems("scripts/test_a.py", "import unittest\n"), [] + ) + + def test_imports_resolve_to_the_tracked_test_modules(self) -> None: + files = frozenset( + { + "scripts/tool.py", + "scripts/test_tool.py", + "scripts/nvx_tools/__init__.py", + "scripts/nvx_tools/helper.py", + "scripts/nvx_tools/test_local.py", + "scripts/powershell_testing.py", + } + ) + for path, source, allowed in ( + ("scripts/tool.py", "from scripts import test_tool\n", False), + ("scripts/tool.py", "from scripts import test_tool as alias\n", False), + ("scripts/nvx_tools/helper.py", "from . import test_local\n", False), + ("scripts/nvx_tools/helper.py", "from .. import test_tool\n", False), + ("scripts/nvx_tools/helper.py", "from .test_local import x\n", False), + ("scripts/tool.py", "from scripts import powershell_testing\n", False), + ("scripts/nvx_tools/helper.py", "from . import helper\n", True), + ("scripts/tool.py", "from nvx_tools.helper import test_helper\n", True), + ("scripts/nvx_tools/helper.py", "from .unittest import x\n", True), + ): + with self.subTest(source=source): + problems = layout.python_problems(path, source, files) + self.assertEqual(problems == [], allowed, problems) + + +class RepositoryTests(unittest.TestCase): + def test_the_repository_follows_the_layout(self) -> None: + layout_found = layout.check_snapshot(layout.WorkTree(layout.REPO_ROOT)) + self.assertEqual([str(problem) for problem in layout_found.problems], []) + + def test_main_tightens_the_allowlist_and_reports_problems(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + files = { + "demo/Cargo.toml": CARGO, + "demo/src/lib.rs": LIBRARY, + "demo/src/a.rs": "#[cfg(test)]\nfn helper() {}\n", + layout.ALLOWLIST: allowlist(seams={"demo/src/a.rs": 1}), + } + for path, text in files.items(): + (root / path).parent.mkdir(parents=True, exist_ok=True) + (root / path).write_text(text, encoding="utf-8", newline="\n") + subprocess.run(["git", "init", "-q"], cwd=root, check=True) + subprocess.run( + ["git", "-c", "core.autocrlf=false", "add", "-A"], + cwd=root, + check=True, + capture_output=True, + ) + + def run(*arguments: str) -> tuple[int, str]: + output = io.StringIO() + with ( + contextlib.redirect_stdout(output), + contextlib.redirect_stderr(io.StringIO()), + ): + status = layout.main(["--root", directory, *arguments]) + return status, output.getvalue() + + status, output = run() + self.assertEqual(status, 1) + self.assertIn("demo/src/a.rs:2: test-only-item:", output) + with patch.dict(os.environ, {"GITHUB_ACTIONS": "true"}): + self.assertIn("::error file=demo/src/a.rs,line=2::", run()[1]) + listed = (root / layout.ALLOWLIST).read_text(encoding="utf-8") + (root / "demo/src/a.rs").write_text( + "#[cfg(test)]\nfn helper( {}\n", encoding="utf-8" + ) + status, output = run("--tighten") + self.assertEqual(status, 1) + self.assertIn("demo/src/a.rs:2: rust-syntax:", output) + self.assertEqual( + (root / layout.ALLOWLIST).read_text(encoding="utf-8"), listed + ) + (root / "demo/src/a.rs").write_text("pub fn f() {}\n", encoding="utf-8") + (root / layout.ALLOWLIST).write_text("{", encoding="utf-8") + status, output = run("--tighten") + self.assertEqual(status, 1) + self.assertIn(f"{layout.ALLOWLIST}: ", output) + self.assertEqual((root / layout.ALLOWLIST).read_text(encoding="utf-8"), "{") + (root / layout.ALLOWLIST).write_text(listed, encoding="utf-8") + self.assertEqual(run()[0], 1) + self.assertEqual(run("--tighten"), (0, "")) + self.assertEqual( + (root / layout.ALLOWLIST).read_text(encoding="utf-8"), allowlist() + ) + + +if __name__ == "__main__": + unittest.main()