From 3acc8025149a7dee23773c0588b570a0642c2dba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=95=D0=B2=D0=B3=D0=B5=D0=BD=D0=B8=D0=B9=20=D0=91=D0=BB?= =?UTF-8?q?=D0=B8=D0=BD=D0=BE=D0=B2?= Date: Tue, 29 Sep 2026 14:10:28 +0300 Subject: [PATCH 01/27] Add GitHub workflows, issue templates, and project configuration --- .github/ISSUE_TEMPLATE/bug_report.md | 32 ++++++ .github/ISSUE_TEMPLATE/documentation.md | 26 +++++ .github/ISSUE_TEMPLATE/feature_request.md | 17 +++ .github/ISSUE_TEMPLATE/question.md | 12 ++ .github/workflows/lint.yml | 63 +++++++++++ .github/workflows/release.yml | 39 +++++++ .github/workflows/tests_and_coverage.yml | 71 ++++++++++++ .gitignore | 20 ++++ LICENSE | 131 ++++++++++++++++++++++ pyproject.toml | 71 ++++++++++++ requirements_dev.txt | 15 +++ tests/__init__.py | 0 throngest/__init__.py | 0 13 files changed, 497 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/documentation.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/ISSUE_TEMPLATE/question.md create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/tests_and_coverage.yml create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 pyproject.toml create mode 100644 requirements_dev.txt create mode 100644 tests/__init__.py create mode 100644 throngest/__init__.py diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..95e7494 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,32 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: bug +assignees: pomponchik + +--- + +## Short description + +Replace this text with a short description of the error and the behavior that you expected to see instead. + + +## Describe the bug in detail + +Please add a test that reproduces the bug (i.e., currently fails): + +```python +def test_your_bug(): + ... +``` + +When writing the test, please ensure compatibility with the [`pytest`](https://docs.pytest.org/) framework. + +If for some reason you cannot describe the error in the test format, describe the steps to reproduce it here. + + +## Environment + - OS: ... + - Python version (the output of the `python --version` command): ... + - Version of this package: ... diff --git a/.github/ISSUE_TEMPLATE/documentation.md b/.github/ISSUE_TEMPLATE/documentation.md new file mode 100644 index 0000000..5f5fdc0 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/documentation.md @@ -0,0 +1,26 @@ +--- +name: Documentation fix +about: Add something to the documentation, delete it, or change it +title: '' +labels: documentation +assignees: pomponchik +--- + +## It's cool that you're here! + +Documentation is an important part of the project; we strive to make it high-quality and keep it up to date. Please adjust this template by outlining your proposal. + + +## Type of action + +What do you want to do: remove something, add something, or change something? + + +## Where? + +Specify which part of the documentation you want to change. For example, the name of an existing documentation section or a line number in `README.md`. + + +## The essence + +Please describe the essence of the proposed change. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..117d79f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,17 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: enhancement +assignees: pomponchik + +--- + +## Short description + +What do you propose and why do you consider it important? + + +## Some details + +If you can, provide code examples that will show how your proposal will work. Also, if you can, indicate which alternative approaches you have considered. And finally, describe how you propose to verify that your idea is implemented correctly, if at all possible. diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md new file mode 100644 index 0000000..6f86494 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question.md @@ -0,0 +1,12 @@ +--- +name: Question or consultation +about: Ask anything about this project +title: '' +labels: question +assignees: pomponchik + +--- + +## Your question + +Here you can freely describe your question about the project. Please read the documentation provided before doing this, and ask the question only if it is not answered there. In addition, please keep in mind that this is a free non-commercial project and user support is optional for its author. Response times are not guaranteed. diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..bd9d29d --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,63 @@ +name: Lint + +on: + push + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13', '3.14', '3.14t', '3.15'] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + # Prerelease CPython ABIs can change; keep 3.15 aligned with current wheels. + allow-prereleases: true + check-latest: ${{ matrix.python-version == '3.15' }} + + - name: Set up uv + uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + + - name: Install dependencies + shell: bash + run: uv pip install --system -r requirements_dev.txt + + - name: Install the library + shell: bash + run: uv pip install --system . + + - name: Run ruff + shell: bash + run: ruff check throngest + + - name: Run ruff for tests + shell: bash + run: ruff check tests + + - name: Run mypy + shell: bash + run: >- + mypy + --show-error-codes + --strict + --disallow-any-decorated + --disallow-any-explicit + --disallow-any-expr + --disallow-any-generics + --disallow-any-unimported + --disallow-subclassing-any + --warn-return-any + throngest + + - name: Run mypy for tests + shell: bash + run: mypy --exclude '^tests/typing/' tests diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..641ef68 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,39 @@ +name: Release + +on: + push: + branches: + - main + +jobs: + pypi-publish: + name: upload release to PyPI + runs-on: ubuntu-latest + # Specifying a GitHub environment is optional, but strongly encouraged + environment: release + permissions: + # IMPORTANT: this permission is mandatory for trusted publishing + id-token: write + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Set up uv + uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + + - name: Install dependencies + shell: bash + run: uv pip install --system -r requirements_dev.txt + + - name: Build the project + shell: bash + run: python -m build . + + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/tests_and_coverage.yml b/.github/workflows/tests_and_coverage.yml new file mode 100644 index 0000000..3f3a0e2 --- /dev/null +++ b/.github/workflows/tests_and_coverage.yml @@ -0,0 +1,71 @@ +name: Tests + +on: + push + +jobs: + build: + + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [macos-latest, ubuntu-latest, windows-latest] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13', '3.14', '3.14t', '3.15'] + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + # Prerelease CPython ABIs can change; keep 3.15 aligned with current wheels. + allow-prereleases: true + check-latest: ${{ matrix.python-version == '3.15' }} + + - name: Set up uv + uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + + - name: Install dependencies + shell: bash + run: uv pip install --system -r requirements_dev.txt + + - name: Install the library + shell: bash + run: uv pip install --system . + + - name: Print all libs + shell: bash + run: uv pip list --system + + - name: Run tests and show coverage on the command line + shell: bash + run: | + pth_file="$(python -c 'import sysconfig; print(sysconfig.get_path("purelib"))')/throng_coverage_process_startup.pth" + printf "import os; os.getenv('COVERAGE_PROCESS_START') and __import__('coverage').process_startup()\n" > "$pth_file" + coverage erase + COVERAGE_PROCESS_START="$PWD/pyproject.toml" coverage run -m pytest -n auto --cache-clear --assert=plain + coverage combine + coverage report -m --fail-under=100 + coverage xml + + - name: Upload coverage to Coveralls + if: runner.os == 'Linux' + env: + COVERALLS_REPO_TOKEN: ${{secrets.COVERALLS_REPO_TOKEN}} + uses: coverallsapp/github-action@v2 + with: + format: cobertura + file: coverage.xml + continue-on-error: true + + - name: Run tests and show the branch coverage on the command line + shell: bash + run: | + pth_file="$(python -c 'import sysconfig; print(sysconfig.get_path("purelib"))')/throng_coverage_process_startup.pth" + printf "import os; os.getenv('COVERAGE_PROCESS_START') and __import__('coverage').process_startup()\n" > "$pth_file" + coverage erase + THRONG_COVERAGE_BRANCH=true COVERAGE_PROCESS_START="$PWD/pyproject.toml" coverage run -m pytest -n auto --cache-clear --assert=plain + coverage combine + coverage report -m --fail-under=100 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..dddf337 --- /dev/null +++ b/.gitignore @@ -0,0 +1,20 @@ +.DS_Store +__pycache__ +venv +.pytest_cache +build +dist +*.egg-info +test.py +.coverage +.coverage.* +.idea +.ruff_cache +.mutmut-cache +.mypy_cache +html +CLAUDE.md +.claude +mutants +planning_features.md +coverage.xml diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..1a71cb6 --- /dev/null +++ b/LICENSE @@ -0,0 +1,131 @@ +# PolyForm Noncommercial License 1.0.0 + + + +## Acceptance + +In order to get any license under these terms, you must agree +to them as both strict obligations and conditions to all +your licenses. + +## Copyright License + +The licensor grants you a copyright license for the +software to do everything you might do with the software +that would otherwise infringe the licensor's copyright +in it for any permitted purpose. However, you may +only distribute the software according to [Distribution +License](#distribution-license) and make changes or new works +based on the software according to [Changes and New Works +License](#changes-and-new-works-license). + +## Distribution License + +The licensor grants you an additional copyright license +to distribute copies of the software. Your license +to distribute covers distributing the software with +changes and new works permitted by [Changes and New Works +License](#changes-and-new-works-license). + +## Notices + +You must ensure that anyone who gets a copy of any part of +the software from you also gets a copy of these terms or the +URL for them above, as well as copies of any plain-text lines +beginning with `Required Notice:` that the licensor provided +with the software. For example: + +> Required Notice: Copyright Yoyodyne, Inc. (http://example.com) + +## Changes and New Works License + +The licensor grants you an additional copyright license to +make changes and new works based on the software for any +permitted purpose. + +## Patent License + +The licensor grants you a patent license for the software that +covers patent claims the licensor can license, or becomes able +to license, that you would infringe by using the software. + +## Noncommercial Purposes + +Any noncommercial purpose is a permitted purpose. + +## Personal Uses + +Personal use for research, experiment, and testing for +the benefit of public knowledge, personal study, private +entertainment, hobby projects, amateur pursuits, or religious +observance, without any anticipated commercial application, +is use for a permitted purpose. + +## Noncommercial Organizations + +Use by any charitable organization, educational institution, +public research organization, public safety or health +organization, environmental protection organization, +or government institution is use for a permitted purpose +regardless of the source of funding or obligations resulting +from the funding. + +## Fair Use + +You may have "fair use" rights for the software under the +law. These terms do not limit them. + +## No Other Rights + +These terms do not allow you to sublicense or transfer any of +your licenses to anyone else, or prevent the licensor from +granting licenses to anyone else. These terms do not imply +any other licenses. + +## Patent Defense + +If you make any written claim that the software infringes or +contributes to infringement of any patent, your patent license +for the software granted under these terms ends immediately. If +your company makes such a claim, your patent license ends +immediately for work on behalf of your company. + +## Violations + +The first time you are notified in writing that you have +violated any of these terms, or done anything with the software +not covered by your licenses, your licenses can nonetheless +continue if you come into full compliance with these terms, +and take practical steps to correct past violations, within +32 days of receiving notice. Otherwise, all your licenses +end immediately. + +## No Liability + +***As far as the law allows, the software comes as is, without +any warranty or condition, and the licensor will not be liable +to you for any damages arising out of these terms or the use +or nature of the software, under any kind of legal claim.*** + +## Definitions + +The **licensor** is the individual or entity offering these +terms, and the **software** is the software the licensor makes +available under these terms. + +**You** refers to the individual or entity agreeing to these +terms. + +**Your company** is any legal entity, sole proprietorship, +or other kind of organization that you work for, plus all +organizations that have control over, are under the control of, +or are under common control with that organization. **Control** +means ownership of substantially all the assets of an entity, +or the power to direct its management and policies by vote, +contract, or otherwise. Control can be direct or indirect. + +**Your licenses** are all the licenses granted to you for the +software under these terms. + +**Use** means anything you do with the software requiring one +of your licenses. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..fbf5a2e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,71 @@ +[build-system] +requires = ["setuptools==68.0.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "throngest" +version = "0.0.1" +authors = [ + { name="Evgeniy Blinov", email="zheni-b@yandex.ru" }, +] +description = 'Distributed pytest execution in isolated environments' +readme = "README.md" +requires-python = ">=3.8" +dependencies = [ + 'throng>=0.0.3', + 'skelet>=0.0.21', +] +classifiers = [ + "Operating System :: OS Independent", + 'Operating System :: MacOS :: MacOS X', + 'Operating System :: Microsoft :: Windows', + 'Operating System :: POSIX', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', + 'Programming Language :: Python :: 3.14', + 'Programming Language :: Python :: 3.15', + 'Programming Language :: Python :: Free Threading', + 'Programming Language :: Python :: Free Threading :: 3 - Stable', + 'License :: OSI Approved :: MIT License', + 'Intended Audience :: Developers', + 'Topic :: Software Development :: Libraries', + 'Typing :: Typed', +] +keywords = [ + 'pytest', +] + +[tool.setuptools.package-data] +"throngest" = ["py.typed"] + +[tool.setuptools.packages.find] +include = ["throngest*"] + +[tool.mutmut] +paths_to_mutate=["throngest"] + +[tool.coverage.run] +branch = "${THRONG_COVERAGE_BRANCH-false}" +omit = ["*tests*"] +parallel = true +plugins = ["coverage_pyver_pragma"] +source = ["throngest"] + +[tool.pytest.ini_options] +norecursedirs = ["build", "mutants"] + +[tool.ruff] +lint.ignore = ['E501', 'E712', 'PTH123', 'PTH118', 'PLR2004', 'PTH107', 'SIM105', 'SIM102', 'RET503', 'PLR0912', 'C901', 'E731', 'F821'] +lint.select = ["ERA001", "YTT", "ASYNC", "BLE", "B", "A", "COM", "INP", "PIE", "T20", "PT", "RSE", "RET", "SIM", "SLOT", "TID252", "ARG", "PTH", "I", "C90", "N", "E", "W", "D201", "D202", "D419", "F", "PL", "PLE", "PLR", "PLW", "RUF", "TRY201", "TRY400", "TRY401"] +lint.isort.combine-as-imports = true +format.quote-style = "single" + +[project.urls] +'Source' = 'https://github.com/mutating/throngest' +'Tracker' = 'https://github.com/mutating/throngest/issues' diff --git a/requirements_dev.txt b/requirements_dev.txt new file mode 100644 index 0000000..75164b5 --- /dev/null +++ b/requirements_dev.txt @@ -0,0 +1,15 @@ +pytest==8.3.5 +pytest-xdist==3.6.1; python_version < '3.9' +pytest-xdist==3.8.0; python_version >= '3.9' +coverage==7.6.1; python_version == '3.8' +coverage==7.6.10; python_version >= '3.9' +coverage-pyver-pragma==0.4.0 +build==1.2.2.post1 +mypy==1.14.1 +pytest-mypy-testing==0.1.3 +ruff==0.14.6 +mutmut==3.2.3 +cosmic-ray==8.3.15; python_version < '3.9' +cosmic-ray==8.4.6; python_version >= '3.9' +full_match==0.0.3 +locklib==0.0.22 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/throngest/__init__.py b/throngest/__init__.py new file mode 100644 index 0000000..e69de29 From c56818f0e0e288dd517028ba2a989e3c16aa7dd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=95=D0=B2=D0=B3=D0=B5=D0=BD=D0=B8=D0=B9=20=D0=91=D0=BB?= =?UTF-8?q?=D0=B8=D0=BD=D0=BE=D0=B2?= Date: Tue, 29 Sep 2026 14:10:52 +0300 Subject: [PATCH 02/27] Fix typo in lint and mypy target directory --- .github/workflows/lint.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index bd9d29d..c4ecdda 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -37,7 +37,7 @@ jobs: - name: Run ruff shell: bash - run: ruff check throngest + run: ruff check throngtest - name: Run ruff for tests shell: bash @@ -56,7 +56,7 @@ jobs: --disallow-any-unimported --disallow-subclassing-any --warn-return-any - throngest + throngtest - name: Run mypy for tests shell: bash From 22e9f74a38e42a399fa5b65ae191bec803b229e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=95=D0=B2=D0=B3=D0=B5=D0=BD=D0=B8=D0=B9=20=D0=91=D0=BB?= =?UTF-8?q?=D0=B8=D0=BD=D0=BE=D0=B2?= Date: Tue, 29 Sep 2026 14:15:18 +0300 Subject: [PATCH 03/27] Add `--isolates=0` to pytest coverage run commands --- .github/workflows/tests_and_coverage.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests_and_coverage.yml b/.github/workflows/tests_and_coverage.yml index 3f3a0e2..a0939c7 100644 --- a/.github/workflows/tests_and_coverage.yml +++ b/.github/workflows/tests_and_coverage.yml @@ -45,7 +45,7 @@ jobs: pth_file="$(python -c 'import sysconfig; print(sysconfig.get_path("purelib"))')/throng_coverage_process_startup.pth" printf "import os; os.getenv('COVERAGE_PROCESS_START') and __import__('coverage').process_startup()\n" > "$pth_file" coverage erase - COVERAGE_PROCESS_START="$PWD/pyproject.toml" coverage run -m pytest -n auto --cache-clear --assert=plain + COVERAGE_PROCESS_START="$PWD/pyproject.toml" coverage run -m pytest --isolates=0 -n auto --cache-clear --assert=plain coverage combine coverage report -m --fail-under=100 coverage xml @@ -66,6 +66,6 @@ jobs: pth_file="$(python -c 'import sysconfig; print(sysconfig.get_path("purelib"))')/throng_coverage_process_startup.pth" printf "import os; os.getenv('COVERAGE_PROCESS_START') and __import__('coverage').process_startup()\n" > "$pth_file" coverage erase - THRONG_COVERAGE_BRANCH=true COVERAGE_PROCESS_START="$PWD/pyproject.toml" coverage run -m pytest -n auto --cache-clear --assert=plain + THRONG_COVERAGE_BRANCH=true COVERAGE_PROCESS_START="$PWD/pyproject.toml" coverage run -m pytest --isolates=0 -n auto --cache-clear --assert=plain coverage combine coverage report -m --fail-under=100 From 0d60bfb763fe841d81cc907a287cbc47a9283749 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=95=D0=B2=D0=B3=D0=B5=D0=BD=D0=B8=D0=B9=20=D0=91=D0=BB?= =?UTF-8?q?=D0=B8=D0=BD=D0=BE=D0=B2?= Date: Tue, 29 Sep 2026 14:16:55 +0300 Subject: [PATCH 04/27] Update locklib to 0.0.25 --- requirements_dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev.txt b/requirements_dev.txt index 75164b5..29bb427 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -12,4 +12,4 @@ mutmut==3.2.3 cosmic-ray==8.3.15; python_version < '3.9' cosmic-ray==8.4.6; python_version >= '3.9' full_match==0.0.3 -locklib==0.0.22 +locklib==0.0.25 From ea6a296f27836cc84a0a06afcf85a04b0eeeda94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=95=D0=B2=D0=B3=D0=B5=D0=BD=D0=B8=D0=B9=20=D0=91=D0=BB?= =?UTF-8?q?=D0=B8=D0=BD=D0=BE=D0=B2?= Date: Tue, 29 Sep 2026 14:17:10 +0300 Subject: [PATCH 05/27] Rename throngest to pytest-throng and add pytest plugin --- pyproject.toml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index fbf5a2e..c8bf40f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools==68.0.0"] build-backend = "setuptools.build_meta" [project] -name = "throngest" +name = "throngtest" version = "0.0.1" authors = [ { name="Evgeniy Blinov", email="zheni-b@yandex.ru" }, @@ -12,6 +12,7 @@ description = 'Distributed pytest execution in isolated environments' readme = "README.md" requires-python = ">=3.8" dependencies = [ + 'pytest>=8.3.5,<10', 'throng>=0.0.3', 'skelet>=0.0.21', ] @@ -41,21 +42,24 @@ keywords = [ 'pytest', ] +[project.entry-points.pytest11] +throngtest = "throngtest.plugin" + [tool.setuptools.package-data] -"throngest" = ["py.typed"] +"throngtest" = ["py.typed"] [tool.setuptools.packages.find] -include = ["throngest*"] +include = ["throngtest*"] [tool.mutmut] -paths_to_mutate=["throngest"] +paths_to_mutate=["throngtest"] [tool.coverage.run] branch = "${THRONG_COVERAGE_BRANCH-false}" omit = ["*tests*"] parallel = true plugins = ["coverage_pyver_pragma"] -source = ["throngest"] +source = ["throngtest"] [tool.pytest.ini_options] norecursedirs = ["build", "mutants"] @@ -67,5 +71,5 @@ lint.isort.combine-as-imports = true format.quote-style = "single" [project.urls] -'Source' = 'https://github.com/mutating/throngest' -'Tracker' = 'https://github.com/mutating/throngest/issues' +'Source' = 'https://github.com/mutating/pytest-throng' +'Tracker' = 'https://github.com/mutating/pytest-throng/issues' From 6045a37211b10d6763408cfd3c814983562ba921 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=95=D0=B2=D0=B3=D0=B5=D0=BD=D0=B8=D0=B9=20=D0=91=D0=BB?= =?UTF-8?q?=D0=B8=D0=BD=D0=BE=D0=B2?= Date: Tue, 29 Sep 2026 14:17:19 +0300 Subject: [PATCH 06/27] Remove empty throngest/__init__.py --- throngest/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 throngest/__init__.py diff --git a/throngest/__init__.py b/throngest/__init__.py deleted file mode 100644 index e69de29..0000000 From d32e4c6c261d81ca56e08201c2fd44847551342a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=95=D0=B2=D0=B3=D0=B5=D0=BD=D0=B8=D0=B9=20=D0=91=D0=BB?= =?UTF-8?q?=D0=B8=D0=BD=D0=BE=D0=B2?= Date: Tue, 29 Sep 2026 14:17:35 +0300 Subject: [PATCH 07/27] Add throngtest package with type marker --- throngtest/__init__.py | 0 throngtest/py.typed | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 throngtest/__init__.py create mode 100644 throngtest/py.typed diff --git a/throngtest/__init__.py b/throngtest/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/throngtest/py.typed b/throngtest/py.typed new file mode 100644 index 0000000..e69de29 From 54b52aa114de4c9bd00cfef9c1417fc7509282d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=95=D0=B2=D0=B3=D0=B5=D0=BD=D0=B8=D0=B9=20=D0=91=D0=BB?= =?UTF-8?q?=D0=B8=D0=BD=D0=BE=D0=B2?= Date: Tue, 29 Sep 2026 15:05:50 +0300 Subject: [PATCH 08/27] Add throngtest plugin to distribute pytest tests across isolates --- throngtest/distribution.py | 54 ++++++++ throngtest/plugin.py | 56 ++++++++ throngtest/protocol.py | 57 ++++++++ throngtest/runner.py | 265 +++++++++++++++++++++++++++++++++++++ throngtest/settings.py | 69 ++++++++++ throngtest/worker.py | 152 +++++++++++++++++++++ throngtest/xdist.py | 86 ++++++++++++ 7 files changed, 739 insertions(+) create mode 100644 throngtest/distribution.py create mode 100644 throngtest/plugin.py create mode 100644 throngtest/protocol.py create mode 100644 throngtest/runner.py create mode 100644 throngtest/settings.py create mode 100644 throngtest/worker.py create mode 100644 throngtest/xdist.py diff --git a/throngtest/distribution.py b/throngtest/distribution.py new file mode 100644 index 0000000..7d5e2f7 --- /dev/null +++ b/throngtest/distribution.py @@ -0,0 +1,54 @@ +"""Deterministic partitions, shared by the controller and the workers.""" + +from difflib import unified_diff +from hashlib import sha256 +from itertools import islice +from typing import Dict, List, Sequence + + +def fingerprint(nodeids: Sequence[str]) -> str: + # Length prefixes make the encoding unambiguous even for unusual node IDs. + digest = sha256() + for nodeid in nodeids: + value = nodeid.encode('utf-8') + digest.update(len(value).to_bytes(8, 'big')) + digest.update(value) + return digest.hexdigest() + + +def collection_difference(expected: Sequence[str], actual: Sequence[str]) -> str: + difference = list(islice(unified_diff(expected, actual, fromfile='controller', tofile='isolate', n=2, lineterm=''), 101)) + if len(difference) > 100: + difference[-1] = '... collection diff truncated after 100 lines' + return '\n'.join([ + 'test collection differs between the controller and the isolate', + 'Possible causes to check:', + ' * Test files were added, removed, or excluded from the isolate snapshot;', + ' preparation commands may generate or change files.', + ' * Parametrization or parameter IDs depend on environment variables, files,', + ' external data, random values, or the current time.', + ' * Collection order is unstable, for example parameters come from a set.', + ' * Test selection differs: check -k, -m, addopts, configuration,', + ' and installed plugins or dependencies.', + ' * Absolute paths in parameter IDs change when the project is copied', + ' to a different directory, even if the tests are otherwise equivalent.', + '', + f'controller: {len(expected)} selected tests', + f'isolate: {len(actual)} selected tests', + *difference, + ]) + + +def partition(nodeids: Sequence[str], workers: int, mode: str) -> List[List[int]]: + if workers < 1: + raise ValueError('workers must be positive') + if mode not in ('tests', 'files'): + raise ValueError('distribution must be tests or files') + groups: Dict[str, List[int]] = {} + for index, nodeid in enumerate(nodeids): + key = nodeid.split('::', 1)[0] if mode == 'files' else str(index) + groups.setdefault(key, []).append(index) + shards: List[List[int]] = [[] for _ in range(min(workers, len(groups)))] + for group in sorted(groups.values(), key=len, reverse=True): + min(shards, key=len).extend(group) + return [sorted(shard) for shard in shards] diff --git a/throngtest/plugin.py b/throngtest/plugin.py new file mode 100644 index 0000000..71ea891 --- /dev/null +++ b/throngtest/plugin.py @@ -0,0 +1,56 @@ +"""Pytest entry point. Distribution is enabled by default with up to four isolates.""" + +from typing import Dict, Generator, List, Optional, cast + +import pytest + +from throngtest.protocol import Request +from throngtest.runner import Runner +from throngtest.settings import ARGUMENTS, WORKER, read_settings +from throngtest.xdist import Shard, suspend + + +def pytest_addoption(parser: pytest.Parser) -> None: + group = parser.getgroup('throngtest', 'Run test subsets in throng isolates') + group.addoption('--isolates', dest='throngtest_workers', metavar='COUNT', default=None, + help='Maximum isolate count; 0 disables distribution (default: 4)') + for name, description in ( + ('backend', 'Throng plugin name (default: temporary_directory)'), + ('distribution', 'Partition by tests or files (default: tests)'), + ('python', 'Python executable available inside the isolate'), + ('exclude', 'JSON list of patterns excluded from the isolate snapshot'), + ('preparation', 'JSON list of commands run in each isolate before pytest (default: [])'), + ): + group.addoption(f'--throngtest-{name}', default=None, help=description) + group.addoption('--throngtest-check-fingerprints', action='store_const', const=True, default=None, + dest='throngtest_check_fingerprints', help='Require identical ordered test collections in all isolates (default: disabled)') + group.addoption('--throngtest-no-check-fingerprints', action='store_const', const=False, default=None, + dest='throngtest_check_fingerprints', help='Disable collection fingerprint checks, overriding environment and TOML settings') + + +def pytest_load_initial_conftests(early_config: pytest.Config, args: List[str]) -> None: + early_config.stash[ARGUMENTS] = list(args) + + +@pytest.hookimpl(wrapper=True, tryfirst=True) # type: ignore[misc] +def pytest_cmdline_main(config: pytest.Config) -> Generator[None, Optional[int], Optional[int]]: + if not config.stash.get(WORKER, False) and not hasattr(config, 'workerinput') and read_settings(config).workers: + suspend(config) + return (yield) + + +@pytest.hookimpl(trylast=True) # type: ignore[misc] # pluggy's decorator bound includes Any. +def pytest_configure(config: pytest.Config) -> None: + workerinput = cast(Optional[Dict[str, object]], getattr(config, 'workerinput', None)) or {} + if 'throngtest' in workerinput: + config.stash[WORKER] = True + config.pluginmanager.register(Shard(Request.unpack(cast(str, workerinput['throngtest']))), 'throngtest-shard') + if config.stash.get(WORKER, False): + return + settings = read_settings(config) + if settings.workers: + if cast(bool, config.getoption('usepdb')): + raise pytest.UsageError('throngtest cannot be combined with --pdb') + if any(cast(object, config.getoption(name, default=False)) for name in ('lf', 'failedfirst', 'newfirst', 'stepwise')): + raise pytest.UsageError('throngtest does not support cache-based selection (--lf/--ff/--nf) or --stepwise') + config.pluginmanager.register(Runner(settings), 'throngtest-runner') diff --git a/throngtest/protocol.py b/throngtest/protocol.py new file mode 100644 index 0000000..05b5e74 --- /dev/null +++ b/throngtest/protocol.py @@ -0,0 +1,57 @@ +"""A versioned, JSON-only protocol carried by throng's command output.""" + +import base64 +import json +from dataclasses import dataclass +from typing import Dict, List, Optional, cast + + +class WorkerError(Exception): + """An isolate failed to produce a complete, trustworthy pytest result.""" + + def __init__(self, message: str, exitcode: int = 3) -> None: + super().__init__(message) + self.exitcode = exitcode + + +def encode(data: Dict[str, object]) -> str: + return base64.b64encode(json.dumps(data, ensure_ascii=True).encode('ascii')).decode('ascii') + + +def decode(value: str) -> Dict[str, object]: + result = cast(object, json.loads(base64.b64decode(value, validate=True))) + if not isinstance(result, dict): + raise ValueError('expected a JSON object') + return cast(Dict[str, object], result) + + +@dataclass +class Request: + arguments: List[str] + fingerprint: Optional[str] + workers: int + distribution: str + shard: int + marker: str + directory: str + + def pack(self) -> str: + return encode(cast(Dict[str, object], vars(self))) + + @classmethod + def unpack(cls, value: str) -> 'Request': + data = decode(value) + return cls(**data) # type: ignore[arg-type] + + +def read_response(stdout: str, marker: str) -> Dict[str, object]: + for line in reversed(stdout.splitlines()): + if line.startswith(marker): + try: + response = decode(line[len(marker):]) + except ValueError as error: + raise WorkerError(f'invalid worker response: {error}') from error + if response.get('version') != 1: + raise WorkerError('unsupported worker protocol version') + return response + raise WorkerError('worker terminated without a result') diff --git a/throngtest/runner.py b/throngtest/runner.py new file mode 100644 index 0000000..2742069 --- /dev/null +++ b/throngtest/runner.py @@ -0,0 +1,265 @@ +"""Coordinate isolates through the public throng manager API.""" + +import os +import shlex +import warnings +from collections import Counter +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path +from typing import Dict, List, Optional, Sequence, cast +from uuid import uuid4 + +import pytest +from cantok import SimpleToken +from throng import AbstractManager, throng + +from throngtest.distribution import collection_difference, fingerprint, partition +from throngtest.protocol import Request, WorkerError, read_response +from throngtest.settings import ARGUMENTS, Settings +from throngtest.xdist import NESTED + + +def relocate(argument: str, root: Path, invocation: Path, directory: Path) -> str: + """Relocate project paths without interpreting pytest's other arguments.""" + if not argument or argument.startswith(('--throngtest-', '--isolates=')) or (argument.startswith('-') and '=' not in argument): + return argument + prefix, separator, value = argument.partition('=') if argument.startswith('-') else ('', '', argument) + path, *selectors = value.split('::') + candidate = Path(path) + if not candidate.is_absolute(): + if invocation == directory: + return argument + candidate = invocation / candidate + if candidate.exists(): + try: + candidate.relative_to(root) + except ValueError: + return argument + value = '::'.join([os.path.relpath(candidate, directory), *selectors]) + return prefix + separator + value + + +def execute(manager: AbstractManager, request: Request, settings: Settings, token: SimpleToken, nodeids: Sequence[str] = ()) -> Dict[str, object]: + command = shlex.join([settings.python, '-c', 'from throngtest.worker import main; import sys; main(sys.argv[1])', request.pack()]) + preparation_output = '' + try: + with manager.scope as isolate: + for index, instruction in enumerate(settings.preparation, 1): + try: + prepared = isolate.run(instruction, token=token) + except Exception as error: + raise WorkerError(f'isolate {request.shard + 1}: preparation command {index} could not execute: {instruction}\n{error}') from error + preparation_output += (prepared.stdout or '') + (prepared.stderr or '') + if prepared.returncode != 0: + raise WorkerError( + f'isolate {request.shard + 1}: preparation command {index} failed ' + f'with exit code {prepared.returncode}: {instruction}', + ) + result = isolate.run(command, token=token) + except Exception as error: # Third-party plugins may raise their own exception types. + raise WorkerError(f'could not execute isolate: {error}\npreparation output:\n{preparation_output}') from error + stdout = result.stdout or '' + try: + response = read_response(stdout, request.marker) + # The decoded protocol is represented by the diagnostic below; avoid + # flooding errors with its base64 payload. Preserve unrelated output. + lines = stdout.splitlines(keepends=True) + response_index = max(index for index, line in enumerate(lines) if line.startswith(request.marker)) + stdout = ''.join(lines[:response_index]) + if stdout.endswith('\n'): + stdout = stdout[:-1] + stdout += ''.join(lines[response_index + 1:]) + if response.get('exitcode') != result.returncode: + raise WorkerError('worker exit code does not match its result') + if response['exitcode'] not in (0, 1): + detail = str(response.get('error', '')) + if 'collection' in response: + collection = response['collection'] + if not isinstance(collection, list) or any(not isinstance(item, str) for item in collection): + raise WorkerError('worker returned a malformed test collection') + detail = collection_difference(nodeids, cast(List[str], collection)) + raise WorkerError(f"worker exited with code {response['exitcode']}: {detail}", exitcode=2 if response['exitcode'] == 2 else 3) + response['output'] = preparation_output + stdout + (result.stderr or '') + return response + except WorkerError as error: + raise WorkerError(f'{error}\npreparation output:\n{preparation_output}\nstdout:\n{stdout}\nstderr:\n{result.stderr or ""}', exitcode=error.exitcode) from error + + +def finished_tests(session: pytest.Session, response: Dict[str, object], expected: List[str], parallel: bool) -> List[str]: + finished = response.get('finished') + if not isinstance(finished, list): + raise WorkerError('worker returned malformed reports') + if any(not isinstance(nodeid, str) for nodeid in finished): + raise WorkerError('worker returned malformed finished tests') + completed = cast(List[str], finished) + matches = Counter(completed) == Counter(expected) if parallel else completed == expected + if not matches: + # With fail-fast pytest is allowed to finish just a prefix of a shard. + maxfail = cast(int, session.config.getoption('maxfail')) + subset = not (Counter(completed) - Counter(expected)) if parallel else completed == expected[:len(completed)] + interrupted = maxfail or (parallel and response.get('worker_crashed') is True) + if not (interrupted and response['exitcode'] == 1 and completed and subset): + raise WorkerError('worker did not execute its assigned tests exactly once') + return completed + + +def replay(session: pytest.Session, response: Dict[str, object], expected: Optional[List[str]], parallel: bool = False) -> None: + if expected is None: + assigned = response.get('assigned') + if not isinstance(assigned, list) or any(not isinstance(item, str) for item in assigned): + raise WorkerError('worker returned malformed assigned tests') + expected = cast(List[str], assigned) + reports = response.get('reports') + if not isinstance(reports, list): + raise WorkerError('worker returned malformed reports') + completed = finished_tests(session, response, expected, parallel) + decoded: List[pytest.TestReport] = [] + for data in cast(List[object], reports): + if not isinstance(data, dict): + raise WorkerError('worker returned a malformed test report') + # JSON has no tuples; pytest's skip representation requires one. + record = cast(Dict[str, object], data) + longrepr = record.get('longrepr') + if isinstance(longrepr, list): + record['longrepr'] = tuple(cast(List[object], longrepr)) + report = cast(object, session.config.hook.pytest_report_from_serializable(config=session.config, data=data)) + if not isinstance(report, pytest.TestReport) or report.nodeid not in expected: # type: ignore[misc] # pytest report constructors accept arbitrary plugin attributes. + raise WorkerError('worker returned an unexpected test report') + decoded.append(report) + endings = [report.nodeid for report in decoded if report.when == 'teardown' or (parallel and report.when == '???')] + if not (Counter(endings) == Counter(completed) if parallel else endings == completed): + raise WorkerError('worker returned an incomplete set of test reports') + if parallel: + decoded = parallel_reports(decoded) + block: List[pytest.TestReport] = [] + for report in decoded: + block.append(report) + if parallel and report.when == '???' and report.failed: + block.clear() + elif report.when == 'teardown': + stages = ['setup', 'call', 'teardown'] if block[0].passed else ['setup', 'teardown'] + if [part.when for part in block] != stages or any(part.nodeid != report.nodeid for part in block): + raise WorkerError('worker returned incomplete test phases') + block.clear() + if block: + raise WorkerError('worker returned unterminated test phases') + if response['exitcode'] == 1 and not any(report.failed for report in decoded) and not session.testsfailed: + raise WorkerError('worker failed without reporting a test failure') + for warning in cast(List[Dict[str, object]], response.get('warnings', [])): + warning_message = warnings.WarningMessage( + f"{warning['category']}: {warning['message']}", pytest.PytestWarning, + cast(str, warning['filename']), cast(int, warning['lineno']), + ) + session.config.hook.pytest_warning_recorded.call_historic(kwargs={ + 'warning_message': warning_message, 'when': 'runtest', + 'nodeid': warning['nodeid'], 'location': None, + }) + for report in decoded: + if report.when == 'setup': + session.config.hook.pytest_runtest_logstart(nodeid=report.nodeid, location=report.location) + session.config.hook.pytest_runtest_logreport(report=report) + if report.when == 'teardown' or (parallel and report.when == '???'): + session.config.hook.pytest_runtest_logfinish(nodeid=report.nodeid, location=report.location) + output = cast(str, response.get('output', '')) + if output: + reporter = session.config.pluginmanager.hasplugin('terminalreporter') + if reporter: + session.config.get_terminal_writer().write(output) + + +def parallel_reports(reports: List[pytest.TestReport]) -> List[pytest.TestReport]: + """Reassemble interleaved xdist phases into complete individual executions.""" + pending: Dict[str, List[pytest.TestReport]] = {} + result: List[pytest.TestReport] = [] + for report in reports: + worker = cast(object, getattr(report, 'worker_id', None)) + if not isinstance(worker, str): + raise WorkerError('xdist report is missing its worker ID') + block = pending.setdefault(worker, []) + block.append(report) + if any(part.nodeid != report.nodeid for part in block): + raise WorkerError('worker returned incomplete test phases') + if report.when in ('teardown', '???'): + if report.when == '???' and (not report.failed or [part.when for part in block[:-1]] not in ([], ['setup'], ['setup', 'call'])): + raise WorkerError('worker returned malformed crash phases') + result.extend(block) + del pending[worker] + if pending: + raise WorkerError('worker returned unterminated test phases') + return result + + +class Runner: + def __init__(self, settings: Settings) -> None: + self.settings = settings + + def pytest_report_header(self) -> str: + return f'throngtest: {self.settings.workers} isolates, backend={self.settings.backend}, distribution={self.settings.distribution}' + + @pytest.hookimpl(tryfirst=True) # type: ignore[misc] # pluggy's decorator exposes Any in its generic bound. + def pytest_runtestloop(self, session: pytest.Session) -> bool: + if session.testsfailed and not cast(bool, session.config.getoption('continue_on_collection_errors')): + raise session.Interrupted(f'{session.testsfailed} errors during collection') + if cast(bool, session.config.getoption('collectonly')) or not session.items: + return True + root = session.config.rootpath + invocation = session.config.invocation_params.dir + try: + directory = invocation.relative_to(root) + except ValueError: + directory = Path() + arguments = [] + preserve_value = False + for argument in session.config.stash[ARGUMENTS]: + if preserve_value: + arguments.append(argument) + preserve_value = False + else: + arguments.append(relocate(argument, root, invocation, root / directory)) + preserve_value = argument == '--isolates' or ( + argument.startswith('--throngtest-') and '=' not in argument and argument not in ( + '--throngtest-check-fingerprints', '--throngtest-no-check-fingerprints', + ) + ) + arguments.extend(['--rootdir', os.path.relpath(root, root / directory)]) + nodeids = [item.nodeid for item in session.items] + shards = partition(nodeids, self.settings.workers, self.settings.distribution) + collection_fingerprint = fingerprint(nodeids) if self.settings.check_fingerprints else None + completed = 0 + token = SimpleToken() + previous = Path.cwd() + try: + # throng's built-in archive backend expects a relative source path; + # local also uses the controller's cwd. Never chdir in worker threads. + os.chdir(root) + managers = throng('.', exclude=self.settings.exclude) + if self.settings.backend not in managers: + raise pytest.UsageError(f'throngtest: unknown backend {self.settings.backend!r}; available: {", ".join(sorted(managers))}') + manager = managers[self.settings.backend] + with ThreadPoolExecutor(max_workers=len(shards), thread_name_prefix='throngtest') as pool: + try: + futures = { + pool.submit(execute, manager, Request( + arguments, collection_fingerprint, len(shards), + self.settings.distribution, index, f'THRONGTEST_{uuid4().hex}:', str(directory), + ), self.settings, token, nodeids): shard + for index, shard in enumerate(shards) + } + for future in as_completed(futures): + response = future.result() + replay(session, response, [nodeids[index] for index in futures[future]] if self.settings.check_fingerprints else None, + session.config.stash.get(NESTED, False) and response.get('parallel') is True) + completed += len(cast(List[str], response['finished'])) + if session.shouldfail: + raise session.Failed(session.shouldfail) + if session.shouldstop: + raise session.Interrupted(session.shouldstop) + finally: + token.cancel() + except WorkerError as error: + pytest.exit(f'throngtest: {error}', returncode=error.exitcode) + finally: + os.chdir(previous) + if not completed: + session.testscollected = 0 + return True diff --git a/throngtest/settings.py b/throngtest/settings.py new file mode 100644 index 0000000..420d33a --- /dev/null +++ b/throngtest/settings.py @@ -0,0 +1,69 @@ +# mypy: disallow_any_expr=False +"""All user configuration is resolved and validated by skelet.""" + +# skelet.Storage currently triggers an unlocated Any-expression error in mypy +# when subclassed. Keep all other strict checks enabled. + +import sys +from typing import List, cast + +import pytest +from skelet import EnvSource, Field, FixedCLISource, MemorySource, Storage, TOMLSource + +WORKER = pytest.StashKey[bool]() +ARGUMENTS = pytest.StashKey[List[str]]() + + +class PytestSource(FixedCLISource[object]): + """Use pytest's argument discovery with skelet's typed CLI conversion.""" + + def __init__(self, config: pytest.Config) -> None: + super().__init__(named_arguments=['workers', 'backend', 'distribution', 'python', 'exclude', 'preparation']) + self.config = config + + def __getitem__(self, key: str) -> str: + value = cast(str, self.config.getoption(f'throngtest_{key}')) + if value is None: + raise KeyError(key) + return value + + +def nonnegative(value: int) -> bool: + return value >= 0 + + +def nonempty(value: str) -> bool: + return bool(value.strip()) + + +def distribution(value: str) -> bool: + return value in ('tests', 'files') + + +def commands(value: List[str]) -> bool: + return all(nonempty(command) for command in value) + + +class Settings(Storage): + workers: int = Field(4, validation={'workers must be nonnegative': nonnegative}) + check_fingerprints: bool = Field(False) + backend: str = Field('temporary_directory', validation={'backend must not be empty': nonempty}) + distribution: str = Field('tests', validation={'distribution must be tests or files': distribution}) + python: str = Field(default_factory=lambda: sys.executable, validation={'python must not be empty': nonempty}) + exclude: List[str] = Field(default_factory=lambda: ['.git/', '.venv/', 'venv/', '__pycache__/', '.pytest_cache/', '.mypy_cache/', '.ruff_cache/', 'build/', 'dist/', 'mutants/']) + preparation: List[str] = Field(default_factory=list, validation={'preparation commands must not be empty': commands}) + + +def read_settings(config: pytest.Config) -> Settings: + try: + check_fingerprints = cast(object, config.getoption('throngtest_check_fingerprints')) + return Settings(_sources=[ + # Pytest supplies native booleans for flags; skelet validates their + # type and resolves priority without treating an absent flag as false. + MemorySource[object]({} if check_fingerprints is None else {'check_fingerprints': check_fingerprints}), + PytestSource(config), + EnvSource[object](prefix='THRONGTEST_'), + TOMLSource[object](config.rootpath / 'pyproject.toml', table='tool.throngtest'), + ]) + except (TypeError, ValueError, OSError) as error: + raise pytest.UsageError(f'throngtest: {error}') from error diff --git a/throngtest/worker.py b/throngtest/worker.py new file mode 100644 index 0000000..644bcff --- /dev/null +++ b/throngtest/worker.py @@ -0,0 +1,152 @@ +"""Runs inside an isolate; never creates further isolates.""" + +import os +import sys +import warnings +from io import StringIO +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Dict, List, Optional, cast + +import pytest +from _pytest._io import TerminalWriter +from _pytest.terminal import TerminalReporter + +from throngtest.distribution import fingerprint, partition +from throngtest.protocol import Request, decode, encode +from throngtest.settings import WORKER +from throngtest.xdist import Node + + +class Worker: + def __init__(self, request: Request, basetemp: str) -> None: + self.request = request + self.basetemp = basetemp + self.config: Optional[pytest.Config] = None + self.reports: List[Dict[str, object]] = [] + self.finished: List[str] = [] + self.error = '' + self.collection: Optional[List[str]] = None + self.assigned: Optional[List[str]] = None + self.warnings: List[Dict[str, object]] = [] + self.parallel = False + self.crashed = False + self.names: Dict[str, str] = {} + + def pytest_load_initial_conftests(self, early_config: pytest.Config) -> None: + early_config.stash[WORKER] = True + + @pytest.hookimpl(optionalhook=True) # type: ignore[misc] + def pytest_configure_node(self, node: Node) -> None: + self.parallel = True + node.workerinput['throngtest'] = self.request.pack() + node.workerinput['throngtest_manifest'] = str(Path(self.basetemp) / f'throngtest-{node.gateway.id}.data') + + @pytest.hookimpl(optionalhook=True) # type: ignore[misc] + def pytest_xdist_node_collection_finished(self, node: Node) -> None: + manifest = decode(Path(cast(str, node.workerinput['throngtest_manifest'])).read_text()) + self.assigned = cast(List[str], manifest['assigned']) + self.names.update(zip(cast(List[str], manifest['ids']), self.assigned)) + + @pytest.hookimpl(optionalhook=True) # type: ignore[misc] + def pytest_testnodedown(self, node: Node, error: object) -> None: + self.crashed |= error is not None + output = cast(Optional[Dict[str, object]], getattr(node, 'workeroutput', None)) or {} + if 'throngtest_collection' in output: + self.collection = cast(List[str], output['throngtest_collection']) + + @pytest.hookimpl(tryfirst=True) # type: ignore[misc] # pluggy's decorator bound includes Any. + def pytest_configure(self, config: pytest.Config) -> None: + self.config = config + # Only the controller writes aggregate output files. Each worker gets + # its own pytest temporary directory even when --basetemp was supplied. + config.option.xmlpath = None + config.option.basetemp = self.basetemp + + @pytest.hookimpl(tryfirst=True) # type: ignore[misc] + def pytest_sessionstart(self, session: pytest.Session) -> None: + reporter = cast(Optional[TerminalReporter], session.config.pluginmanager.get_plugin('terminalreporter')) + if reporter is not None: + # Assertion formatting still needs the terminal writer. Keep the + # reporter registered, but discard its duplicate session output. + reporter._tw = TerminalWriter(StringIO()) + + @pytest.hookimpl(trylast=True) # type: ignore[misc] + def pytest_collection_finish(self, session: pytest.Session) -> None: + nodeids = [item.nodeid for item in session.items] + if self.request.fingerprint is not None and fingerprint(nodeids) != self.request.fingerprint: + self.error = 'test collection differs between the controller and the isolate' + self.collection = nodeids + raise pytest.UsageError(self.error) + shards = partition(nodeids, self.request.workers, self.request.distribution) + indices = shards[self.request.shard] if self.request.shard < len(shards) else [] + self.assigned = [nodeids[index] for index in indices] + session.items[:] = [session.items[index] for index in indices] + session.testscollected = len(session.items) + + def pytest_runtest_logreport(self, report: pytest.TestReport) -> None: + assert self.config is not None + data = cast(Dict[str, object], self.config.hook.pytest_report_to_serializable(config=self.config, report=report)) + node = data.pop('node', None) + if node is not None: + data['worker_id'] = cast(Node, node).gateway.id + if report.when == '???': + self.finished.append(report.nodeid) + self.reports.append(data) + + def pytest_collectreport(self, report: pytest.CollectReport) -> None: + if report.failed: + self.error += str(report.longrepr) + '\n' + + def pytest_runtest_logfinish(self, nodeid: str) -> None: + self.finished.append(nodeid) + + def pytest_warning_recorded(self, warning_message: warnings.WarningMessage, when: str, nodeid: str) -> None: + if when == 'runtest': + self.warnings.append({ + 'message': str(warning_message.message), + 'category': warning_message.category.__name__, + 'filename': warning_message.filename, + 'lineno': warning_message.lineno, + 'nodeid': nodeid, + }) + + +def main(value: str) -> None: + request = Request.unpack(value) + os.chdir(request.directory) + # Effective arguments already include ini addopts and PYTEST_ADDOPTS. + os.environ.pop('PYTEST_ADDOPTS', None) + with TemporaryDirectory(prefix='throngtest-pytest-') as basetemp: + worker = Worker(request, basetemp) + status = pytest.main([*request.arguments, '-o', 'addopts='], plugins=[worker]) + if status == pytest.ExitCode.NO_TESTS_COLLECTED and worker.assigned == []: + status = pytest.ExitCode.OK + if worker.parallel and worker.error and not worker.reports: + status = pytest.ExitCode.INTERRUPTED + if worker.collection is not None: + status = pytest.ExitCode.USAGE_ERROR + if worker.parallel: + for report in worker.reports: + nodeid = cast(str, report['nodeid']) + report['nodeid'] = worker.names.get(nodeid, nodeid) + worker.finished = [worker.names.get(nodeid, nodeid) for nodeid in worker.finished] + if status == pytest.ExitCode.INTERRUPTED and worker.config is not None and cast(int, worker.config.getoption('maxfail')) and any(report['outcome'] == 'failed' for report in worker.reports): + status = pytest.ExitCode.TESTS_FAILED + response: Dict[str, object] = { + 'version': 1, + 'exitcode': int(status), + 'reports': worker.reports, + 'finished': worker.finished, + 'assigned': worker.assigned, + 'error': worker.error, + 'warnings': worker.warnings, + 'worker_crashed': worker.crashed, + 'parallel': worker.parallel, + } + if worker.collection is not None: + response['collection'] = worker.collection + # A leading newline also handles tests that print without a newline. + sys.stdout.write('\n' + request.marker + encode(response) + '\n') + sys.stdout.flush() + raise SystemExit(status) diff --git a/throngtest/xdist.py b/throngtest/xdist.py new file mode 100644 index 0000000..6179fa0 --- /dev/null +++ b/throngtest/xdist.py @@ -0,0 +1,86 @@ +"""Optional cooperation with user-installed xdist, without importing it.""" + +from pathlib import Path +from typing import Dict, Generator, List, Optional, Protocol, Set, cast + +import pytest + +from throngtest.distribution import fingerprint, partition +from throngtest.protocol import Request, encode + +NESTED = pytest.StashKey[bool]() + + +class Gateway(Protocol): + id: str + + +class Node(Protocol): + workerinput: Dict[str, object] + workeroutput: Dict[str, object] + gateway: Gateway + + +def suspend(config: pytest.Config) -> None: + """Leave xdist's original arguments for the isolate, suppress its outer runner.""" + if cast(bool, config.getoption('looponfail', default=False)): + raise pytest.UsageError('throngtest: --looponfail cannot run inside isolates') + processes = cast(object, config.getoption('numprocesses', default=None)) + mode = cast(str, config.getoption('dist', default='no')) + targets = cast(Optional[List[str]], config.getoption('tx', default=None)) or [] + active = bool(processes) or (processes is None and bool(targets) and (mode != 'no' or cast(bool, config.getoption('distload', default=False)))) + config.stash[NESTED] = active + if not active: + return + if mode == 'each': + raise pytest.UsageError('throngtest: --dist=each repeats tests on every xdist worker; use a distributing scheduler such as --dist=load') + if targets or cast(object, config.getoption('px', default=None)): + raise pytest.UsageError('throngtest: nested xdist supports local workers configured with -n; --tx and --px are not supported inside isolates') + config.option.numprocesses = 0 + config.option.dist = 'no' + config.option.distload = False + + +class Shard: + """Select the isolate's subset before xdist advertises its collection.""" + + def __init__(self, request: Request) -> None: + self.request = request + self.original: Dict[int, str] = {} + + @pytest.hookimpl(wrapper=True, tryfirst=True) # type: ignore[misc] + def pytest_collection_modifyitems(self, session: pytest.Session, config: pytest.Config, items: List[pytest.Item]) -> Generator[None, object, object]: + grouped = cast(bool, config.getoption('loadgroup', default=False)) + # Let collection modifiers finish before xdist adds its group suffix. + # Reuse xdist's own hook so both old single-group and new multi-group + # formats work without duplicating its naming rules. + config.option.loadgroup = False + try: + result = yield + finally: + config.option.loadgroup = grouped + if grouped: + self.original = {id(item): item.nodeid for item in items} + others = [plugin for plugin in cast(Set[object], config.pluginmanager.get_plugins()) if type(plugin).__name__ != 'WorkerInteractor'] + config.pluginmanager.subset_hook_caller('pytest_collection_modifyitems', remove_plugins=others)(session=session, config=config, items=items) + return result + + @pytest.hookimpl(tryfirst=True) # type: ignore[misc] + def pytest_collection_finish(self, session: pytest.Session) -> None: + grouped = cast(bool, session.config.getoption('loadgroup', default=False)) + nodeids = [self.original[id(item)] if grouped else item.nodeid for item in session.items] + if self.request.fingerprint is not None and fingerprint(nodeids) != self.request.fingerprint: + output = cast(Dict[str, object], getattr(session.config, 'workeroutput', None)) + output['throngtest_collection'] = nodeids + raise pytest.UsageError('test collection differs between the controller and the isolate') + shards = partition(nodeids, self.request.workers, self.request.distribution) + indices = shards[self.request.shard] if self.request.shard < len(shards) else [] + assigned = [nodeids[index] for index in indices] + session.items[:] = [session.items[index] for index in indices] + session.testscollected = len(session.items) + workerinput = cast(Dict[str, object], getattr(session.config, 'workerinput', None)) + # Local xdist workers share the isolate filesystem. Publish the mapping + # before xdist announces collection, so it survives even a setup crash. + Path(cast(str, workerinput['throngtest_manifest'])).write_text(encode({ + 'assigned': assigned, 'ids': [item.nodeid for item in session.items], + })) From cc59a67eba9bd172c437ea0669394f7d907306f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=95=D0=B2=D0=B3=D0=B5=D0=BD=D0=B8=D0=B9=20=D0=91=D0=BB?= =?UTF-8?q?=D0=B8=D0=BD=D0=BE=D0=B2?= Date: Tue, 29 Sep 2026 16:04:16 +0300 Subject: [PATCH 09/27] Add integration, unit, and xdist tests --- tests/conftest.py | 21 ++ tests/test_distribution.py | 97 ++++++++ tests/test_fingerprints.py | 127 +++++++++++ tests/test_integration.py | 437 +++++++++++++++++++++++++++++++++++++ tests/test_preparation.py | 208 ++++++++++++++++++ tests/test_protocol.py | 34 +++ tests/test_runner.py | 250 +++++++++++++++++++++ tests/test_settings.py | 124 +++++++++++ tests/test_xdist.py | 403 ++++++++++++++++++++++++++++++++++ 9 files changed, 1701 insertions(+) create mode 100644 tests/conftest.py create mode 100644 tests/test_distribution.py create mode 100644 tests/test_fingerprints.py create mode 100644 tests/test_integration.py create mode 100644 tests/test_preparation.py create mode 100644 tests/test_protocol.py create mode 100644 tests/test_runner.py create mode 100644 tests/test_settings.py create mode 100644 tests/test_xdist.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..a301053 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,21 @@ +import os +from pathlib import Path + +import pytest + +pytest_plugins = ['pytester'] + + +@pytest.fixture(autouse=True) +def clean_environment(monkeypatch: pytest.MonkeyPatch) -> None: + for name in os.environ: + if name.startswith('THRONGTEST_'): + monkeypatch.delenv(name) + monkeypatch.delenv('PYTEST_ADDOPTS', raising=False) + if os.environ.get('COVERAGE_PROCESS_START'): + monkeypatch.setenv('COVERAGE_FILE', str(Path(__file__).resolve().parents[1] / '.coverage')) + + +@pytest.fixture(params=['local', 'temporary_directory']) +def backend(request: pytest.FixtureRequest) -> str: + return str(request.param) diff --git a/tests/test_distribution.py b/tests/test_distribution.py new file mode 100644 index 0000000..d87c269 --- /dev/null +++ b/tests/test_distribution.py @@ -0,0 +1,97 @@ +import pytest + +from throngtest.distribution import collection_difference, fingerprint, partition + + +@pytest.mark.parametrize('mode', ['tests', 'files']) +@pytest.mark.parametrize('workers', [1, 2, 3, 20]) +def test_partition_is_complete_and_disjoint(mode: str, workers: int) -> None: + """Assign every collected occurrence exactly once with stable ordering. + + Repeated node IDs must remain distinct by collection index. Requesting more + workers than items also checks that partitioning never creates empty shards. + """ + nodes = ['a.py::one', 'a.py::two', 'b.py::three', 'c.py::four', 'a.py::two'] + shards = partition(nodes, workers, mode) + assert sorted(index for shard in shards for index in shard) == list(range(len(nodes))) + assert all(shard == sorted(shard) for shard in shards) + assert all(shards) + assert len(shards) <= workers + assert shards == partition(nodes, workers, mode) + if mode == 'files': + assert any(all(index in shard for index in (0, 1, 4)) for shard in shards) + + +def test_balanced_partitions() -> None: + """Balance shards by tests per file or round-robin assignment, including empty input.""" + assert partition(['a::1', 'a::2', 'a::3', 'b::1', 'c::1', 'c::2'], 2, 'files') == [[0, 1, 2], [3, 4, 5]] + assert partition(['a', 'b', 'c', 'd', 'e'], 2, 'tests') == [[0, 2, 4], [1, 3]] + assert partition([], 3, 'tests') == [] + + +@pytest.mark.parametrize(('workers', 'mode', 'message'), [(0, 'tests', 'positive'), (-1, 'files', 'positive'), (1, 'bad', 'distribution')]) +def test_invalid_partition(workers: int, mode: str, message: str) -> None: + """Reject nonpositive worker counts and unknown distribution modes.""" + with pytest.raises(ValueError, match=message): + partition(['a'], workers, mode) + + +def test_fingerprint_includes_order_duplicates_and_boundaries() -> None: + """Distinguish collection order, repeated IDs, and identifier boundaries in hashes.""" + values = [[], ['a'], ['a', 'a'], ['a', 'b'], ['b', 'a'], ['ab'], ['a\nb'], ['a', '\nb'], ['тест']] + assert len({fingerprint(value) for value in values}) == len(values) + assert fingerprint(['a', 'b']) == fingerprint(['a', 'b']) + + +@pytest.mark.parametrize(('expected', 'actual', 'changes'), [ + (['a'], ['a', 'b'], ['+b']), + (['a', 'b'], ['a'], ['-b']), + (['a', 'b'], ['b', 'a'], ['+b', '-b']), + (['a', 'a'], ['a'], ['-a']), + ([], ['a'], ['+a']), + (['a'], [], ['-a']), + (['test.py::test_item[/project/файл]'], ['test.py::test_item[/temporary/файл]'], ['-test.py::test_item[/project/файл]', '+test.py::test_item[/temporary/файл]']), +]) +def test_collection_difference_preserves_identifiers_order_and_duplicates(expected: list, actual: list, changes: list) -> None: + """Show exact collection changes, including ordering, duplicate IDs, and Unicode paths.""" + diagnostic = collection_difference(expected, actual) + assert f'controller: {len(expected)} selected tests' in diagnostic + assert f'isolate: {len(actual)} selected tests' in diagnostic + assert '--- controller\n+++ isolate\n@@ ' in diagnostic + lines = diagnostic.splitlines() + assert [line for line in lines if line.startswith(('+', '-')) and line not in ('--- controller', '+++ isolate')] == changes + + +def test_collection_difference_explains_possible_causes_before_the_diff() -> None: + """Place troubleshooting hints before collection counts and the detailed diff.""" + diagnostic = collection_difference(['test.py::old'], ['test.py::new']) + hints, details = diagnostic.split('\ncontroller:', 1) + assert hints.startswith('test collection differs between the controller and the isolate\nPossible causes to check:\n') + for cause in ( + 'Test files were added, removed, or excluded', + 'preparation commands may generate or change files', + 'environment variables, files', + 'external data, random values, or the current time', + 'Collection order is unstable', + 'parameters come from a set', + 'check -k, -m, addopts, configuration', + 'installed plugins or dependencies', + 'Absolute paths in parameter IDs change', + 'even if the tests are otherwise equivalent', + ): + assert cause in hints + assert hints.count(' * ') == 5 + assert details.startswith(' 1 selected tests\nisolate: 1 selected tests\n--- controller\n+++ isolate\n') + + +def test_large_collection_difference_is_bounded() -> None: + """Truncate a large collection diff while retaining counts and troubleshooting hints.""" + expected = [f'test.py::old[{index}]' for index in range(1000)] + actual = [f'test.py::new[{index}]' for index in range(1000)] + lines = collection_difference(expected, actual).splitlines() + diff_start = lines.index('--- controller') + assert lines[diff_start - 2:diff_start] == ['controller: 1000 selected tests', 'isolate: 1000 selected tests'] + assert 'Possible causes to check:' in lines[:diff_start] + assert any('Absolute paths in parameter IDs change' in line for line in lines[:diff_start]) + assert len(lines[diff_start:]) == 101 + assert lines[-1] == '... collection diff truncated after 100 lines' diff --git a/tests/test_fingerprints.py b/tests/test_fingerprints.py new file mode 100644 index 0000000..e415035 --- /dev/null +++ b/tests/test_fingerprints.py @@ -0,0 +1,127 @@ +from pathlib import Path +from typing import Tuple +from xml.etree import ElementTree + +import pytest + + +@pytest.mark.parametrize('configuration', [ + ('default', False), ('cli', True), ('cli', False), + ('environment', True), ('environment', False), ('toml', True), ('toml', False), +]) +def test_changed_identifiers_with_each_setting_source(pytester: pytest.Pytester, backend: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, configuration: Tuple[str, bool]) -> None: + """Reject changed IDs only when fingerprint checking is enabled by a settings source. + + A collection hook appends a suffix only inside isolates. External marker + files verify execution counts, while JUnit checks that modified test IDs + survive reporting when fingerprint checking is disabled. + """ + source, checked = configuration + pytester.makeconftest(''' + from throngtest.settings import WORKER + def pytest_collection_modifyitems(config, items): + if config.stash.get(WORKER, False): + for item in items: + item._nodeid += '[isolate]' + ''') + pytester.makepyfile(f''' + from pathlib import Path + import pytest + @pytest.mark.parametrize('index', range(4)) + def test_item(index): + marker = Path({str(tmp_path)!r}) / str(index) + assert not marker.exists() + marker.touch() + ''') + arguments = ['--isolates=2', f'--throngtest-backend={backend}', '--junitxml=results.xml'] + if source == 'cli': + arguments.append('--throngtest-check-fingerprints' if checked else '--throngtest-no-check-fingerprints') + elif source == 'environment': + monkeypatch.setenv('THRONGTEST_CHECK_FINGERPRINTS', str(checked).lower()) + elif source == 'toml': + pytester.makepyprojecttoml('[tool.throngtest]\ncheck_fingerprints = ' + str(checked).lower()) + result = pytester.runpytest_subprocess(*arguments, timeout=30) + output = result.stdout.str() + result.stderr.str() + if checked: + assert result.ret == pytest.ExitCode.INTERNAL_ERROR + assert 'Possible causes to check:' in output + assert 'controller: 4 selected tests\nisolate: 4 selected tests' in output + assert '[isolate]' in output + assert list(tmp_path.iterdir()) == [] + else: + result.assert_outcomes(passed=4) + assert 'collection differs' not in output + assert len(list(tmp_path.iterdir())) == 4 + cases = ElementTree.parse(pytester.path / 'results.xml').findall('.//testcase') + assert len(cases) == 4 + assert all(case.attrib['name'].endswith('[isolate]') for case in cases) + + +@pytest.mark.parametrize('checked', [False, True]) +def test_fingerprint_flag_before_an_absolute_test_path(pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch, checked: bool) -> None: + """Parse a fingerprint flag separately from the following absolute test path. + + Pytest is invoked from a child directory to exercise path relocation. A + marker file keeps that directory present in the temporary project copy. + """ + pytester.makeini('[pytest]') + test = pytester.makepyfile('def test_ok(): pass') + child = pytester.path / 'child' + child.mkdir() + (child / 'keep.txt').write_text('Keep the invocation directory in the isolate snapshot.') + monkeypatch.chdir(child) + flag = '--throngtest-check-fingerprints' if checked else '--throngtest-no-check-fingerprints' + pytester.runpytest_subprocess(flag, str(test), timeout=30).assert_outcomes(passed=1) + + +def test_default_skips_hashing_and_accepts_absolute_parameter_paths(pytester: pytest.Pytester) -> None: + """Skip fingerprint computation by default even when isolation changes parameter IDs. + + Both hashing entry points are replaced with functions that raise, so a + passing run proves hashing was skipped. A cwd-derived parameter changes + its ID naturally when the project is copied into an isolate. + """ + pytester.makeconftest(''' + import throngtest.runner + import throngtest.worker + def unexpected_hash(*args): + raise AssertionError('fingerprints must not be computed when disabled') + throngtest.runner.fingerprint = unexpected_hash + throngtest.worker.fingerprint = unexpected_hash + ''') + pytester.makepyfile(''' + from pathlib import Path + import pytest + @pytest.mark.parametrize('path', [str(Path.cwd() / 'missing-parent' / 'missing-command')]) + def test_path(path): + assert Path(path).parent.parent == Path.cwd() + ''') + pytester.runpytest_subprocess(timeout=30).assert_outcomes(passed=1) + + +@pytest.mark.parametrize('distribution', ['tests', 'files']) +@pytest.mark.parametrize(('controller_count', 'worker_count'), [(1, 4), (4, 1), (4, 0)]) +def test_changed_collection_sizes_without_fingerprints(pytester: pytest.Pytester, backend: str, distribution: str, controller_count: int, worker_count: int) -> None: + """Run the isolate's collection when unchecked collection sizes differ. + + A hook independently truncates controller and worker collections to exercise + growth, shrinkage, and an empty worker collection under both distributions. + """ + pytester.makeconftest(f''' + from throngtest.settings import WORKER + def pytest_collection_modifyitems(config, items): + count = {worker_count} if config.stash.get(WORKER, False) else {controller_count} + items[:] = items[:count] + ''') + pytester.makepyfile(**{f'test_{index}': 'def test_ok(): pass' for index in range(4)}) + result = pytester.runpytest_subprocess(f'--throngtest-backend={backend}', f'--throngtest-distribution={distribution}', timeout=30) + result.assert_outcomes(passed=worker_count) + assert result.ret == (pytest.ExitCode.OK if worker_count else pytest.ExitCode.NO_TESTS_COLLECTED) + + +@pytest.mark.parametrize('checked', [False, True]) +def test_duplicate_identifiers_remain_valid_in_both_modes(pytester: pytest.Pytester, backend: str, checked: bool) -> None: + """Preserve duplicate test occurrences with fingerprint checking enabled or disabled.""" + test = pytester.makepyfile('def test_ok(): pass') + flag = '--throngtest-check-fingerprints' if checked else '--throngtest-no-check-fingerprints' + pytester.runpytest_subprocess(f'--throngtest-backend={backend}', flag, '--keep-duplicates', str(test), str(test), timeout=30).assert_outcomes(passed=2) diff --git a/tests/test_integration.py b/tests/test_integration.py new file mode 100644 index 0000000..0d79482 --- /dev/null +++ b/tests/test_integration.py @@ -0,0 +1,437 @@ +import json +import os +from pathlib import Path +from xml.etree import ElementTree + +import pytest + + +def run(pytester: pytest.Pytester, backend: str, *arguments: str) -> pytest.RunResult: + return pytester.runpytest_subprocess('--isolates=2', f'--throngtest-backend={backend}', *arguments, timeout=30) + + +@pytest.mark.parametrize(('arguments', 'workers'), [((), 4), (('--isolates=2',), 2), (('--isolates', '2'), 2), (('--throngtest-check-fingerprints',), 4)]) +def test_real_isolates_execute_every_test_once(pytester: pytest.Pytester, backend: str, tmp_path: Path, arguments: tuple, workers: int) -> None: + """Execute every test once through the selected throng backend and isolate count. + + Wrappers observe the real backend run methods, while external files record + test execution, PIDs, and working directories. Together they distinguish + actual isolate dispatch from ordinary pytest execution and expose whether + temporary copies are removed or the local backend shares the project. + """ + pytester.makeconftest(f''' + from pathlib import Path + from uuid import uuid4 + from throng.extensions.local.isolate import LocalIsolate + from throng.extensions.temporary_directory.isolate import TemporaryDirectoryIsolate + def observe(cls): + original = cls.run + def traced(self, *args, **kwargs): + (Path({str(tmp_path)!r}) / ('dispatch-' + uuid4().hex)).write_text(cls.__name__) + return original(self, *args, **kwargs) + cls.run = traced + observe(LocalIsolate) + observe(TemporaryDirectoryIsolate) + ''') + pytester.makepyfile(f''' + import json, os + from pathlib import Path + import pytest + @pytest.mark.parametrize('index', range(8)) + def test_item(index): + assert os.getpid() != {os.getpid()} + target = Path({str(tmp_path)!r}) / str(index) + assert not target.exists() + target.write_text(json.dumps({{'pid': os.getpid(), 'cwd': str(Path.cwd())}})) + Path('isolate-marker').write_text('written') + ''') + result = pytester.runpytest_subprocess(f'--throngtest-backend={backend}', *arguments, timeout=30) + result.assert_outcomes(passed=8) + records = [json.loads((tmp_path / str(index)).read_text()) for index in range(8)] + dispatched = [path.read_text() for path in tmp_path.glob('dispatch-*')] + expected_class = 'LocalIsolate' if backend == 'local' else 'TemporaryDirectoryIsolate' + assert dispatched == [expected_class] * workers + assert len({record['pid'] for record in records}) == workers + directories = {record['cwd'] for record in records} + if backend == 'local': + assert directories == {str(pytester.path.resolve())} + assert (pytester.path / 'isolate-marker').exists() + else: + assert len(directories) == workers + assert all(not Path(directory).exists() for directory in directories) + assert not (pytester.path / 'isolate-marker').exists() + + +def test_temporary_isolates_overlap(pytester: pytest.Pytester, tmp_path: Path) -> None: + """Run temporary-directory isolates concurrently. + + Each process writes to a shared external barrier and waits for both PIDs. + Sequential execution cannot pass this rendezvous; the deadline bounds a + stalled run without using total execution time as the concurrency assertion. + """ + pytester.makepyfile(f''' + import os, time + from pathlib import Path + import pytest + @pytest.mark.parametrize('index', range(2)) + def test_overlap(index): + barrier = Path({str(tmp_path)!r}) + (barrier / str(os.getpid())).touch() + deadline = time.monotonic() + 10 + while len(list(barrier.iterdir())) != 2: + assert time.monotonic() < deadline, 'isolates did not run concurrently' + time.sleep(0.01) + ''') + run(pytester, 'temporary_directory').assert_outcomes(passed=2) + + +def test_files_stay_together_and_fixtures_work(pytester: pytest.Pytester, backend: str, tmp_path: Path) -> None: + """Keep each file in one isolate and run its module fixture setup and teardown once. + + Module fixtures write uniquely named external markers and their PIDs. + Repeated setup fails immediately, while teardown markers and distinct PIDs + confirm fixture completion and distribution across processes. + """ + source = f''' + import os + from pathlib import Path + import pytest + @pytest.fixture(scope='module') + def resource(): + path = Path({str(tmp_path)!r}) / (Path(__file__).stem + '.fixture') + assert not path.exists() + path.write_text(str(os.getpid())) + yield 42 + path.with_suffix('.done').touch() + @pytest.mark.parametrize('index', range(4)) + def test_item(resource, index): + assert resource == 42 + ''' + pytester.makepyfile(test_a=source, test_b=source) + run(pytester, backend, '--throngtest-distribution=files').assert_outcomes(passed=8) + assert (tmp_path / 'test_a.fixture').read_text() != (tmp_path / 'test_b.fixture').read_text() + assert len(list(tmp_path.glob('*.done'))) == 2 + + +def test_reports_and_junit(pytester: pytest.Pytester, backend: str) -> None: + """Preserve pytest outcomes, failure details, captured output, and JUnit properties. + + A generated suite mixes passes, failures, skips, xfail/xpass, and fixture + errors in setup and teardown. JUnit uses xunit1 to retain record_property + metadata, and its case count checks that replay does not duplicate tests. + """ + pytester.makepyfile(''' + import pytest + def test_pass(record_property): + record_property('custom', 'value') + def test_fail(): + print('captured output') + assert 1 == 2 + @pytest.mark.skip(reason='skip reason') + def test_skip(): pass + @pytest.mark.xfail(reason='expected failure') + def test_xfail(): assert False + @pytest.mark.xfail(reason='unexpected success') + def test_xpass(): pass + @pytest.fixture + def broken(): raise RuntimeError('fixture failed') + def test_error(broken): pass + @pytest.fixture + def bad_teardown(): + yield + raise RuntimeError('teardown failed') + def test_teardown(bad_teardown): pass + ''') + result = run(pytester, backend, '--junitxml=results.xml', '-o', 'junit_family=xunit1', '-ra') + result.assert_outcomes(passed=2, failed=1, skipped=1, xfailed=1, xpassed=1, errors=2) + assert result.ret == pytest.ExitCode.TESTS_FAILED + result.stdout.fnmatch_lines_random(['*captured output*', '*fixture failed*', '*teardown failed*', '*assert 1 == 2*']) + assert 'get_terminal_writer' not in result.stdout.str() + xml = ElementTree.parse(pytester.path / 'results.xml') + assert len(xml.findall('.//testcase')) == 7 + assert xml.find('.//property[@name="custom"]') is not None + + +def test_selection_and_conftest(pytester: pytest.Pytester, backend: str) -> None: + """Apply keyword and marker selection while making conftest fixtures available in isolates.""" + pytester.makeconftest(''' + import pytest + @pytest.fixture + def resource(): return 9 + ''') + pytester.makepyfile(''' + import pytest + @pytest.mark.chosen + @pytest.mark.parametrize('value', [1, 2, 3]) + def test_selected(resource, value): assert resource == 9 + def test_ignored(): assert False + ''') + run(pytester, backend, '-k', 'not 2', '-m', 'chosen').assert_outcomes(passed=2, deselected=2) + + +def test_disabled_is_normal_pytest(pytester: pytest.Pytester) -> None: + """Leave ordinary pytest execution active when --isolates=0 disables the runner.""" + pytester.makeconftest(''' + def pytest_sessionstart(session): + assert not session.config.pluginmanager.hasplugin('throngtest-runner') + ''') + pytester.makepyfile('def test_ok(): pass') + pytester.runpytest_subprocess('--isolates=0').assert_outcomes(passed=1) + + +def test_collect_only_does_not_execute(pytester: pytest.Pytester, backend: str) -> None: + """Collect tests without executing their bodies under --collect-only.""" + pytester.makepyfile('def test_never(): raise AssertionError("executed")') + result = run(pytester, backend, '--collect-only') + assert result.ret == 0 + result.stdout.fnmatch_lines(['*1 test collected*']) + + +def test_empty_collection(pytester: pytest.Pytester, backend: str) -> None: + """Return pytest's no-tests exit code for an empty project.""" + assert run(pytester, backend).ret == pytest.ExitCode.NO_TESTS_COLLECTED + + +def test_collection_error(pytester: pytest.Pytester, backend: str) -> None: + """Interrupt the run and display the original error when collection fails.""" + pytester.makepyfile('raise RuntimeError("cannot collect")') + result = run(pytester, backend) + assert result.ret == pytest.ExitCode.INTERRUPTED + result.stdout.fnmatch_lines(['*cannot collect*']) + + +def test_failfast(pytester: pytest.Pytester, backend: str) -> None: + """Stop after early failures when pytest's -x option is enabled. + + Two isolates may already be executing when the first failure arrives, so + the allowed failure count includes one in-flight test from each isolate. + """ + pytester.makepyfile(''' + import pytest + @pytest.mark.parametrize('index', range(10)) + def test_fail(index): assert False + ''') + result = run(pytester, backend, '-x') + assert result.ret == pytest.ExitCode.TESTS_FAILED + assert 1 <= result.parseoutcomes()['failed'] <= 2 + + +def test_crashed_worker_is_not_success(pytester: pytest.Pytester, backend: str) -> None: + """Treat an isolate process exiting without a protocol response as an internal error. + + os._exit bypasses pytest hooks and normal interpreter shutdown, reproducing + a missing worker result rather than a normally reported test failure. + """ + pytester.makepyfile('import os\ndef test_crash(): os._exit(17)') + result = run(pytester, backend) + assert result.ret == pytest.ExitCode.INTERNAL_ERROR + assert 'worker terminated without a result' in result.stdout.str() + result.stderr.str() + + +def test_collection_mismatch(pytester: pytest.Pytester, backend: str, tmp_path: Path) -> None: + """Reject changed collections before execution and show actionable mismatch diagnostics. + + The controller's import creates an external marker that makes later worker + imports generate an extra parameter. This produces a deterministic mismatch + on both backends, with no dependence on random values or process timing. + """ + marker = tmp_path / 'collected' + pytester.makepyfile(f''' + from pathlib import Path + import pytest + marker = Path({str(marker)!r}) + values = [1, 2] if marker.exists() else [1] + marker.touch() + @pytest.mark.parametrize('value', values) + def test_item(value): raise AssertionError('test must not execute') + ''') + result = run(pytester, backend, '--throngtest-check-fingerprints') + assert result.ret == pytest.ExitCode.INTERNAL_ERROR + output = result.stdout.str() + result.stderr.str() + assert 'collection differs' in output + assert 'controller: 1 selected tests' in output + assert 'isolate: 2 selected tests' in output + assert output.index('Possible causes to check:') < output.index('controller: 1 selected tests') + assert 'preparation commands may generate or change files' in output + assert '+test_collection_mismatch.py::test_item[2]' in output + assert 'THRONGTEST_' not in output + + +def test_collection_diagnostic_for_absolute_parameter_path(pytester: pytest.Pytester) -> None: + """Explain fingerprint mismatches caused solely by relocated absolute parameter paths. + + A parameter derives its ID from the current directory, so a temporary copy + changes the identifier without changing the collection size. The diff must + expose both paths while omitting the encoded worker response. + """ + pytester.makepyfile(''' + from pathlib import Path + import pytest + @pytest.mark.parametrize('value', [str(Path.cwd() / 'missing-command')]) + def test_item(value): raise AssertionError('test must not execute') + ''') + result = run(pytester, 'temporary_directory', '--throngtest-check-fingerprints') + assert result.ret == pytest.ExitCode.INTERNAL_ERROR + output = result.stdout.str() + result.stderr.str() + assert 'controller: 1 selected tests\nisolate: 1 selected tests' in output + assert output.index('Absolute paths in parameter IDs change') < output.index('controller: 1 selected tests') + assert 'even if the tests are otherwise equivalent' in output + expected = str(pytester.path.resolve() / 'missing-command') + assert f'-test_collection_diagnostic_for_absolute_parameter_path.py::test_item[{expected}]' in output + assert '+test_collection_diagnostic_for_absolute_parameter_path.py::test_item[' in output + assert f'+test_collection_diagnostic_for_absolute_parameter_path.py::test_item[{expected}]' not in output + assert 'THRONGTEST_' not in output + + +def test_warning_and_uncaptured_output(pytester: pytest.Pytester, backend: str) -> None: + """Forward worker stdout, stderr, and a single warning when pytest capture is disabled.""" + pytester.makepyfile(''' + import sys, warnings + def test_output(): + print('raw stdout marker') + print('raw stderr marker', file=sys.stderr) + warnings.warn('runtime warning marker', UserWarning) + ''') + result = run(pytester, backend, '-s') + result.assert_outcomes(passed=1, warnings=1) + result.stdout.fnmatch_lines_random(['*raw stdout marker*', '*raw stderr marker*', '*runtime warning marker*']) + + +def test_ini_and_environment_addopts(pytester: pytest.Pytester, backend: str, monkeypatch: pytest.MonkeyPatch) -> None: + """Combine selection options from pytest.ini and PYTEST_ADDOPTS in isolate runs.""" + pytester.makeini('[pytest]\naddopts = -m chosen\nmarkers = chosen: selected test') + monkeypatch.setenv('PYTEST_ADDOPTS', '-k good') + pytester.makepyfile(''' + import pytest + @pytest.mark.chosen + def test_good(): pass + @pytest.mark.chosen + def test_bad(): assert False + def test_good_unmarked(): assert False + ''') + run(pytester, backend).assert_outcomes(passed=1, deselected=2) + + +def test_enable_using_skelet_sources(pytester: pytest.Pytester, backend: str, monkeypatch: pytest.MonkeyPatch) -> None: + """Configure isolate execution through TOML and environment values without CLI options.""" + pytester.makepyprojecttoml(f'[tool.throngtest]\nworkers = 1\nbackend = "{backend}"\ndistribution = "files"') + monkeypatch.setenv('THRONGTEST_WORKERS', '2') + pytester.makepyfile(test_a='def test_a(): pass', test_b='def test_b(): pass') + result = pytester.runpytest_subprocess() + result.assert_outcomes(passed=2) + result.stdout.fnmatch_lines([f'*throngtest: 2 isolates, backend={backend}, distribution=files*']) + + +@pytest.mark.parametrize('inside', [True, False]) +def test_invocation_directory_and_absolute_nodeid(pytester: pytest.Pytester, backend: str, monkeypatch: pytest.MonkeyPatch, inside: bool) -> None: + """Select an absolute node ID when invoked inside or outside the project root. + + The test path includes a space, and an unselected failing test catches + accidental loss of the node selector during path relocation. + """ + pytester.makeini('[pytest]') + child = pytester.path / 'sub directory' + child.mkdir() + test = child / 'test_example.py' + test.write_text('def test_ok(): pass\ndef test_bad(): assert False\n') + monkeypatch.chdir(child if inside else pytester.path.parent) + # pytester.runpytest_subprocess executes from the current working directory. + result = run(pytester, backend, str(test) + '::test_ok') + result.assert_outcomes(passed=1) + + +def test_missing_interpreter(pytester: pytest.Pytester, backend: str) -> None: + """Report a missing configured Python executable as a worker startup failure.""" + pytester.makepyfile('def test_ok(): pass') + result = run(pytester, backend, '--throngtest-python=missing-throngtest-python') + assert result.ret == pytest.ExitCode.INTERNAL_ERROR + assert 'worker terminated without a result' in result.stdout.str() + result.stderr.str() + + +def test_unknown_backend(pytester: pytest.Pytester) -> None: + """Reject unknown backends with a usage error listing the built-in alternatives.""" + pytester.makepyfile('def test_ok(): pass') + result = run(pytester, 'unknown') + assert result.ret == pytest.ExitCode.USAGE_ERROR + result.stderr.fnmatch_lines(['*unknown backend*local*temporary_directory*']) + + +@pytest.mark.parametrize('option', ['--pdb', '--lf', '--ff', '--nf', '--sw']) +def test_incompatible_execution_options(pytester: pytest.Pytester, option: str) -> None: + """Reject debugging and cache-dependent execution options incompatible with isolates.""" + pytester.makepyfile('def test_ok(): pass') + result = run(pytester, 'local', option) + assert result.ret == pytest.ExitCode.USAGE_ERROR + + +def test_duplicate_nodeids(pytester: pytest.Pytester, backend: str) -> None: + """Execute both requested occurrences when the same test file is collected twice.""" + test = pytester.makepyfile('def test_ok(): pass') + run(pytester, backend, '--keep-duplicates', str(test), str(test)).assert_outcomes(passed=2) + + +def test_excluded_file_is_not_copied(pytester: pytest.Pytester) -> None: + """Omit explicitly excluded project files from temporary isolate copies.""" + (pytester.path / 'private.txt').write_text('not for the snapshot') + pytester.makepyfile('from pathlib import Path\ndef test_excluded(): assert not Path("private.txt").exists()') + run(pytester, 'temporary_directory', '--throngtest-exclude=["private.txt"]').assert_outcomes(passed=1) + + +def test_strict_xpass_is_failure(pytester: pytest.Pytester, backend: str) -> None: + """Count a strict unexpected pass as a failure and preserve pytest's failure exit code.""" + pytester.makepyfile('import pytest\n@pytest.mark.xfail(strict=True)\ndef test_ok(): pass') + result = run(pytester, backend) + result.assert_outcomes(failed=1) + assert result.ret == pytest.ExitCode.TESTS_FAILED + + +def test_without_terminal_reporter(pytester: pytest.Pytester, backend: str) -> None: + """Complete isolate execution successfully with pytest's terminal reporter disabled.""" + pytester.makepyfile('def test_ok(): print("output without terminal")') + assert run(pytester, backend, '-p', 'no:terminal', '-s', '--assert=plain').ret == pytest.ExitCode.OK + + +def test_failfast_cancels_running_isolate_and_removes_copies(pytester: pytest.Pytester, tmp_path: Path) -> None: + """Cancel an active sibling isolate on failfast and remove both temporary copies. + + The failing test waits for the slow test's external start marker before + failing. A marker after the slow sleep detects missed cancellation, while + externally recorded paths let the controller check cleanup after shutdown. + """ + pytester.makepyfile(f''' + import time + from pathlib import Path + import pytest + observer = Path({str(tmp_path)!r}) + def test_fail(): + (observer / 'fast-directory').write_text(str(Path.cwd())) + deadline = time.monotonic() + 10 + while not (observer / 'started').exists(): + assert time.monotonic() < deadline + time.sleep(0.01) + pytest.fail('cancel the other isolate') + def test_slow(): + (observer / 'slow-directory').write_text(str(Path.cwd())) + (observer / 'started').touch() + time.sleep(10) + (observer / 'not-cancelled').touch() + ''') + result = run(pytester, 'temporary_directory', '-x') + assert result.ret == pytest.ExitCode.TESTS_FAILED + assert not (tmp_path / 'not-cancelled').exists() + assert not Path((tmp_path / 'fast-directory').read_text()).exists() + assert not Path((tmp_path / 'slow-directory').read_text()).exists() + + +def test_keyboard_interrupt_in_worker(pytester: pytest.Pytester, backend: str) -> None: + """Propagate a worker KeyboardInterrupt as pytest's interrupted exit code.""" + pytester.makepyfile('def test_interrupt(): raise KeyboardInterrupt') + assert run(pytester, backend).ret == pytest.ExitCode.INTERRUPTED + + +def test_continue_after_collection_errors(pytester: pytest.Pytester, backend: str) -> None: + """Run collectable tests while retaining collection errors when continuation is requested.""" + pytester.makepyfile(test_broken='raise RuntimeError("broken collection")', test_good='def test_ok(): pass') + result = run(pytester, backend, '--continue-on-collection-errors') + result.assert_outcomes(passed=1, errors=1) + assert result.ret == pytest.ExitCode.TESTS_FAILED diff --git a/tests/test_preparation.py b/tests/test_preparation.py new file mode 100644 index 0000000..d1e567d --- /dev/null +++ b/tests/test_preparation.py @@ -0,0 +1,208 @@ +import json +import shlex +import sys +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest +from cantok import SimpleToken + +from throngtest.protocol import Request, WorkerError, encode +from throngtest.runner import execute +from throngtest.settings import Settings + + +def python_command(source: str) -> str: + return shlex.join([sys.executable, '-c', source]) + + +@pytest.mark.parametrize('source', ['cli', 'cli_separate', 'environment', 'toml']) +def test_preparation_runs_in_order_in_every_isolate(pytester: pytest.Pytester, backend: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, source: str) -> None: + """Run preparation once per isolate, in order, before starting its pytest process. + + Wrappers trace real backend calls per isolate, and the second command + depends on the first command's marker. Invoking pytest from a subdirectory + also checks preparation at the project root and test execution from the + invocation directory, with preparation output forwarded to the controller. + """ + commands = [ + python_command("from pathlib import Path; Path('first marker').write_text('ready'); print('preparation stdout')"), + python_command("import sys; from pathlib import Path; assert Path('first marker').read_text() == 'ready'; Path('second marker').write_text('prepared'); print('preparation stderr', file=sys.stderr)"), + ] + pytester.makeini('[pytest]') + pytester.makeconftest(f''' + import json + from pathlib import Path + from uuid import uuid4 + from throng.extensions.local.isolate import LocalIsolate + from throng.extensions.temporary_directory.isolate import TemporaryDirectoryIsolate + from throngtest.settings import WORKER + + def observe(cls): + original = cls.run + def traced(self, command, *args, **kwargs): + if not hasattr(self, '_trace_name'): + self._trace_name = uuid4().hex + path = Path({str(tmp_path)!r}) / self._trace_name + with path.open('a') as stream: + stream.write(json.dumps([cls.__name__, command]) + '\\n') + return original(self, command, *args, **kwargs) + cls.run = traced + observe(LocalIsolate) + observe(TemporaryDirectoryIsolate) + + def pytest_sessionstart(session): + if session.config.stash.get(WORKER, False): + assert (session.config.rootpath / 'second marker').read_text() == 'prepared' + ''') + child = pytester.path / 'nested' + child.mkdir() + (child / 'test_example.py').write_text(''' +from pathlib import Path +import pytest +@pytest.mark.parametrize('index', range(4)) +def test_prepared(index): + assert (Path.cwd().parent / 'second marker').read_text() == 'prepared' +''') + arguments = ['--isolates=2', f'--throngtest-backend={backend}'] + if source == 'cli': + arguments.append('--throngtest-preparation=' + json.dumps(commands)) + elif source == 'cli_separate': + arguments.extend(['--throngtest-preparation', json.dumps(commands)]) + elif source == 'environment': + monkeypatch.setenv('THRONGTEST_PREPARATION', json.dumps(commands)) + else: + pytester.makepyprojecttoml('[tool.throngtest]\npreparation = ' + json.dumps(commands)) + monkeypatch.chdir(child) + result = pytester.runpytest_subprocess(*arguments, timeout=30) + result.assert_outcomes(passed=4) + assert result.stdout.str().count('preparation stdout') == 2 + assert result.stdout.str().count('preparation stderr') == 2 + traces = [path.read_text().splitlines() for path in tmp_path.iterdir()] + assert len(traces) == 2 + expected_class = 'LocalIsolate' if backend == 'local' else 'TemporaryDirectoryIsolate' + for trace in traces: + calls = [json.loads(line) for line in trace] + assert len(calls) == 3 + assert [call[0] for call in calls] == [expected_class] * 3 + assert [call[1] for call in calls[:2]] == commands + assert 'from throngtest.worker import main' in calls[2][1] + assert (pytester.path / 'second marker').exists() == (backend == 'local') + + +def test_preparation_failure_stops_commands_and_tests(pytester: pytest.Pytester, backend: str, tmp_path: Path) -> None: + """Stop on the first preparation failure, retain diagnostics, and clean up the isolate. + + A shared marker would be created by either the remaining command or the + test, so its absence checks both execution barriers. Another external file + records the isolate directory for the cleanup assertion after failure. + """ + directory = tmp_path / 'directory' + forbidden = tmp_path / 'must-not-run' + commands = [ + python_command(f"from pathlib import Path; Path({str(directory)!r}).write_text(str(Path.cwd())); print('earlier preparation output')"), + python_command("import sys; print('failed preparation stdout'); print('failed preparation stderr', file=sys.stderr); sys.exit(7)"), + python_command(f'from pathlib import Path; Path({str(forbidden)!r}).touch()'), + ] + pytester.makepyfile(f'from pathlib import Path\ndef test_never(): Path({str(forbidden)!r}).touch()') + result = pytester.runpytest_subprocess('--isolates=1', f'--throngtest-backend={backend}', '--throngtest-preparation=' + json.dumps(commands), timeout=30) + assert result.ret == pytest.ExitCode.INTERNAL_ERROR + output = result.stdout.str() + result.stderr.str() + assert 'preparation command 2 failed with exit code 7' in output + assert commands[1] in output + assert 'earlier preparation output' in output + assert 'failed preparation stdout' in output + assert 'failed preparation stderr' in output + assert not forbidden.exists() + assert Path(directory.read_text()).exists() == (backend == 'local') + + +@pytest.mark.parametrize(('arguments', 'exitcode'), [(('--isolates=0',), pytest.ExitCode.OK), (('--collect-only',), pytest.ExitCode.OK), (('-k', 'not test_ok'), pytest.ExitCode.NO_TESTS_COLLECTED)]) +def test_preparation_is_not_run_without_isolates(pytester: pytest.Pytester, tmp_path: Path, arguments: tuple, exitcode: pytest.ExitCode) -> None: + """Skip preparation when isolates are disabled, only collecting, or no tests are selected.""" + marker = tmp_path / 'must-not-run' + commands = [python_command(f'from pathlib import Path; Path({str(marker)!r}).touch()')] + pytester.makepyfile('def test_ok(): pass') + result = pytester.runpytest_subprocess(*arguments, '--throngtest-preparation=' + json.dumps(commands), timeout=30) + assert result.ret == exitcode + assert not marker.exists() + + +def test_preparation_failure_cancels_other_preparation(pytester: pytest.Pytester, tmp_path: Path) -> None: + """Cancel another isolate's active preparation and remove both copies after a failure. + + An atomic mkdir elects one command to fail only after the other signals it + has started sleeping. External markers reveal uncancelled completion and + preserve both directory paths for checking cleanup after the run. + """ + pytester.makepyfile('def test_a(): pass\ndef test_b(): pass') + script = pytester.makepyfile(prepare=f''' + import sys, time + from pathlib import Path + observer = Path({str(tmp_path)!r}) + try: + (observer / 'leader').mkdir() + except FileExistsError: + (observer / 'slow-directory').write_text(str(Path.cwd())) + (observer / 'started').touch() + time.sleep(10) + (observer / 'not-cancelled').touch() + else: + (observer / 'fast-directory').write_text(str(Path.cwd())) + deadline = time.monotonic() + 10 + while not (observer / 'started').exists(): + assert time.monotonic() < deadline + time.sleep(0.01) + sys.exit(9) + ''') + commands = [shlex.join([sys.executable, script.name])] + result = pytester.runpytest_subprocess('--isolates=2', '--throngtest-preparation=' + json.dumps(commands), timeout=30) + assert result.ret == pytest.ExitCode.INTERNAL_ERROR + assert 'preparation command 1 failed with exit code 9' in result.stdout.str() + result.stderr.str() + assert not (tmp_path / 'not-cancelled').exists() + assert not Path((tmp_path / 'fast-directory').read_text()).exists() + assert not Path((tmp_path / 'slow-directory').read_text()).exists() + + +@pytest.mark.parametrize('failure', [RuntimeError('preparation backend error'), SimpleNamespace(stdout='worker crashed', stderr='', returncode=7)]) +def test_preparation_cleanup_on_backend_error_and_worker_crash(failure: object) -> None: + """Preserve preparation output and clean up after backend errors or a missing worker result. + + A mock first returns successful preparation output, then either raises in + the next preparation command or returns a worker exit without a response. + Every dispatched command must receive the same cancellation token. + """ + manager = MagicMock() + isolate = manager.scope.__enter__.return_value + isolate.run.side_effect = [SimpleNamespace(stdout='preparation output', stderr=None, returncode=0), failure] + token = SimpleToken() + commands = ['prepare', 'prepare again'] if isinstance(failure, Exception) else ['prepare'] + settings = Settings(_sources=[]) + settings.preparation = commands + with pytest.raises(WorkerError) as caught: + execute(manager, Request([], '', 1, 'tests', 0, 'marker:', '.'), settings, token) + assert isolate.run.call_count == 2 + assert all(call.kwargs['token'] is token for call in isolate.run.call_args_list) + manager.scope.__exit__.assert_called_once() + assert 'preparation output' in str(caught.value) + if isinstance(failure, Exception): + assert 'preparation command 2 could not execute: prepare again' in str(caught.value) + assert 'preparation backend error' in str(caught.value) + else: + assert 'worker terminated without a result' in str(caught.value) + + +def test_preparation_accepts_commands_without_output() -> None: + """Accept successful preparation whose stdout and stderr are both None.""" + manager = MagicMock() + isolate = manager.scope.__enter__.return_value + isolate.run.side_effect = [ + SimpleNamespace(stdout=None, stderr=None, returncode=0), + SimpleNamespace(stdout='marker:' + encode({'version': 1, 'exitcode': 0}), stderr=None, returncode=0), + ] + settings = Settings(_sources=[]) + settings.preparation = ['prepare'] + result = execute(manager, Request([], '', 1, 'tests', 0, 'marker:', '.'), settings, SimpleToken()) + assert result['output'] == '' + manager.scope.__exit__.assert_called_once() diff --git a/tests/test_protocol.py b/tests/test_protocol.py new file mode 100644 index 0000000..6870180 --- /dev/null +++ b/tests/test_protocol.py @@ -0,0 +1,34 @@ +import base64 +from typing import Optional + +import pytest + +from throngtest.protocol import Request, WorkerError, decode, encode, read_response + + +@pytest.mark.parametrize('fingerprint', [None, 'fingerprint']) +def test_round_trip_and_unrelated_stdout(fingerprint: Optional[str]) -> None: + """Round-trip requests and extract Unicode responses from unrelated process output.""" + request = Request(['-k', 'русский "test"'], fingerprint, 3, 'files', 1, 'unique:', '.') + assert Request.unpack(request.pack()) == request + response = {'version': 1, 'reports': [], 'message': '🌏'} + output = 'unrelated\nunique:' + encode(response) + '\nmore unrelated output\n' + assert read_response(output, 'unique:') == response + + +def test_decoder_rejects_non_object() -> None: + """Reject valid JSON that is not a protocol object.""" + with pytest.raises(ValueError, match='JSON object'): + decode(base64.b64encode(b'[]').decode()) + + +@pytest.mark.parametrize(('text', 'message'), [ + ('no result', 'without a result'), + ('marker:broken!', 'invalid worker response'), + ('marker:' + base64.b64encode(b'\xff').decode('ascii'), 'invalid worker response'), + ('marker:' + encode({'version': 2}), 'protocol version'), +]) +def test_invalid_response(text: str, message: str) -> None: + """Diagnose missing responses, malformed encoding, and unsupported protocol versions.""" + with pytest.raises(WorkerError, match=message): + read_response(text, 'marker:') diff --git a/tests/test_runner.py b/tests/test_runner.py new file mode 100644 index 0000000..7f0581c --- /dev/null +++ b/tests/test_runner.py @@ -0,0 +1,250 @@ +from pathlib import Path +from types import SimpleNamespace +from typing import Dict, List, Literal, cast +from unittest.mock import MagicMock, Mock + +import pytest +from cantok import SimpleToken +from throng import AbstractManager + +from throngtest.protocol import Request, WorkerError, encode +from throngtest.runner import Runner, execute, relocate, replay +from throngtest.settings import ARGUMENTS, Settings + + +def test_relocate_paths(tmp_path: Path) -> None: + """Relocate project paths while preserving node selectors and unrelated arguments. + + A file named '2' exposes accidental relocation of --isolates=2. The long + preparation value checks that command text is not treated as a filesystem path. + """ + root = tmp_path / 'project' + root.mkdir() + test = root / 'test.py' + test.touch() + assert relocate('--rootdir=' + str(root), root, tmp_path, root) == '--rootdir=.' + assert relocate(str(test) + '::test_a[x y]', root, tmp_path, root) == 'test.py::test_a[x y]' + assert relocate('project/test.py', root, tmp_path, root) == 'test.py' + assert relocate(str(tmp_path), root, root, root) == str(tmp_path) + assert relocate('-x', root, root, root) == '-x' + assert relocate('', root, root, root) == '' + assert relocate('missing', root, tmp_path, root) == 'missing' + child = root / 'subdir' + child.mkdir() + (child / '2').touch() + assert relocate('--isolates=2', root, child, root) == '--isolates=2' + preparation = '--throngtest-preparation=["echo ' + 'long command ' * 100 + '"]' + assert relocate(preparation, root, tmp_path, root) == preparation + + +@pytest.mark.parametrize('stage', ['enter', 'run', 'exit']) +def test_isolate_errors_are_reported_and_cleanup_attempted(stage: str) -> None: + """Wrap backend failures and attempt cleanup after successfully entering a scope. + + The mocked context manager fails separately on entry, execution, and exit + to check the cleanup obligations at each point in the isolate lifecycle. + """ + manager = Mock() + scope = manager.scope + scope.__enter__ = Mock() + scope.__exit__ = Mock(return_value=False) + isolate = scope.__enter__.return_value + target = {'enter': scope.__enter__, 'run': isolate.run, 'exit': scope.__exit__}[stage] + target.side_effect = RuntimeError('backend problem') + with pytest.raises(WorkerError, match='could not execute isolate: backend problem'): + execute(manager, Request([], '', 1, 'tests', 0, 'marker:', '.'), Settings(_sources=[]), SimpleToken()) + if stage != 'enter': + scope.__exit__.assert_called_once() + + +def test_exit_code_mismatch() -> None: + """Reject a worker response whose exit code contradicts the process exit status.""" + manager = MagicMock() + manager.scope.__enter__.return_value.run.return_value = SimpleNamespace(stdout='marker:' + encode({'version': 1, 'exitcode': 0}), stderr='diagnostic', returncode=1) + with pytest.raises(WorkerError, match='exit code does not match'): + execute(manager, Request([], '', 1, 'tests', 0, 'marker:', '.'), Settings(_sources=[]), SimpleToken()) + manager.scope.__exit__.assert_called_once() + + +def test_collection_diagnostic_retains_output_without_encoded_response() -> None: + """Show collection changes and process output without exposing the protocol payload. + + The mocked stdout surrounds an encoded collection response with ordinary + output, checking that filtering removes only protocol data and blank lines. + """ + manager = MagicMock() + data: Dict[str, object] = {'version': 1, 'exitcode': 4, 'collection': ['test.py::new']} + manager.scope.__enter__.return_value.run.return_value = SimpleNamespace( + stdout='before\n\nmarker:' + encode(data) + '\nafter\n', stderr='stderr diagnostic', returncode=4, + ) + with pytest.raises(WorkerError) as caught: + execute(manager, Request([], '', 1, 'tests', 0, 'marker:', '.'), Settings(_sources=[]), SimpleToken(), nodeids=['test.py::old']) + diagnostic = str(caught.value) + assert caught.value.exitcode == 3 + assert 'controller: 1 selected tests\nisolate: 1 selected tests' in diagnostic + assert '-test.py::old\n+test.py::new' in diagnostic + assert 'stdout:\nbefore\nafter\n' in diagnostic + assert 'stderr diagnostic' in diagnostic + assert 'marker:' not in diagnostic + assert encode(data) not in diagnostic + + +@pytest.mark.parametrize('collection', ['not a list', [42]]) +def test_malformed_collection_is_reported(collection: object) -> None: + """Reject worker collection data that is not a list of string identifiers.""" + manager = MagicMock() + manager.scope.__enter__.return_value.run.return_value = SimpleNamespace( + stdout='marker:' + encode({'version': 1, 'exitcode': 4, 'collection': collection}), stderr='', returncode=4, + ) + with pytest.raises(WorkerError, match='malformed test collection'): + execute(manager, Request([], '', 1, 'tests', 0, 'marker:', '.'), Settings(_sources=[]), SimpleToken()) + + +def report_data(when: Literal['setup', 'call', 'teardown'], outcome: Literal['passed', 'failed', 'skipped'] = 'passed', nodeid: str = 'test.py::test_a') -> Dict[str, object]: + report = pytest.TestReport(nodeid, ('test.py', 0, 'test_a'), {}, outcome, None, when) + return report._to_json() + + +@pytest.fixture +def session(pytester: pytest.Pytester) -> pytest.Session: + config = pytester.parseconfigure() + session = pytest.Session.from_config(config) + # Exercise real pytest report serialization, without terminal session state. + config.pluginmanager.unregister(name='terminalreporter') + return session + + +@pytest.mark.parametrize('checked', [False, True]) +@pytest.mark.parametrize(('change', 'message'), [ + ({'finished': None}, 'malformed reports'), + ({'reports': None}, 'malformed reports'), + ({'finished': []}, 'exactly once'), + ({'finished': ['other']}, 'exactly once'), + ({'reports': [None]}, 'malformed test report'), + ({'reports': [dict(report_data('call'), nodeid='other')]}, 'unexpected test report'), + ({'reports': [report_data('call')]}, 'incomplete set'), + ({'reports': [report_data('teardown')]}, 'incomplete test phases'), + ({'reports': [report_data('setup'), report_data('call'), report_data('teardown'), report_data('setup')]}, 'unterminated'), + ({'exitcode': 1}, 'without reporting a test failure'), +]) +def test_corrupted_or_incomplete_reports_never_pass(session: pytest.Session, change: Dict[str, object], message: str, checked: bool) -> None: + """Reject corrupted results regardless of whether fingerprints are checked. + + Each case alters one field in an otherwise complete result. Reports use + pytest's own JSON representation, with its report-type discriminator added + explicitly so deserialization reaches the intended consistency check. + """ + data: Dict[str, object] = { + 'exitcode': 0, 'finished': ['test.py::test_a'], + 'assigned': ['test.py::test_a'], + 'reports': [report_data('setup'), report_data('call'), report_data('teardown')], + } + data.update(change) + # _to_json is the payload before pytest adds its discriminator. + for report in cast(List[object], data['reports'] or []): + if isinstance(report, dict): + report['$report_type'] = 'TestReport' + with pytest.raises(WorkerError, match=message): + replay(session, data, ['test.py::test_a'] if checked else None) + + +@pytest.mark.parametrize('assigned', [None, 'not a list', [42]]) +def test_unchecked_reports_still_require_valid_assignment(session: pytest.Session, assigned: object) -> None: + """Validate worker-assigned identifiers even when controller fingerprints are disabled.""" + with pytest.raises(WorkerError, match='malformed assigned tests'): + replay(session, {'assigned': assigned}, None) + + +def test_interrupt_cancels_token_and_restores_cwd(pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """Cancel dispatched work and restore the invocation directory on a requested stop. + + A stub executor records the real cancellation token, and report replay is + bypassed to isolate runner cleanup. The invocation directory deliberately + differs from the project root so restoration is observable. + """ + config = pytester.parseconfigure() + session = pytest.Session.from_config(config) + session.items = [cast(pytest.Item, SimpleNamespace(nodeid='test.py::test_a'))] + session.shouldstop = 'requested stop' + config.stash[ARGUMENTS] = [] + token_seen: List[SimpleToken] = [] + + def fake_execute(_manager: AbstractManager, _request: Request, _settings: Settings, token: SimpleToken, _nodeids: List[str]) -> Dict[str, object]: + token_seen.append(token) + return {'finished': ['test.py::test_a']} + + monkeypatch.setattr('throngtest.runner.execute', fake_execute) + monkeypatch.setattr('throngtest.runner.replay', lambda *_args: None) + monkeypatch.chdir(tmp_path) + settings = Settings(_sources=[]) + settings.workers = 1 + with pytest.raises(session.Interrupted, match='requested stop'): + Runner(settings).pytest_runtestloop(session) + assert Path.cwd() == tmp_path + assert len(token_seen) == 1 + assert not token_seen[0] + + +def xdist_report(nodeid: str, when: str, worker: str = 'gw0', outcome: str = 'passed') -> Dict[str, object]: + return {'nodeid': nodeid, 'when': when, 'worker_id': worker, 'outcome': outcome, + 'location': ('test.py', 0, 'test'), 'keywords': {}, 'longrepr': None, '$report_type': 'TestReport'} + + +def test_parallel_reports_accept_interleaving_and_completion_order(session: pytest.Session) -> None: + """Accept interleaved reports and completion order independent of assignment order. + + Synthetic reports make gw1 finish its test between gw0's setup and call, + exercising per-worker phase tracking without depending on process timing. + """ + reports = [xdist_report('a', 'setup'), xdist_report('b', 'setup', 'gw1'), + xdist_report('b', 'call', 'gw1'), xdist_report('b', 'teardown', 'gw1'), + xdist_report('a', 'call'), xdist_report('a', 'teardown')] + replay(session, {'exitcode': 0, 'finished': ['b', 'a'], 'reports': reports}, ['a', 'b'], parallel=True) + assert session.testsfailed == 0 + + +@pytest.mark.parametrize('finished', [[], ['a', 'a'], ['a', 'unexpected']]) +def test_parallel_finished_tests_cannot_omit_or_duplicate_assignments(session: pytest.Session, finished: List[str]) -> None: + """Reject successful parallel results with missing, repeated, or unassigned completions.""" + with pytest.raises(WorkerError, match='exactly once'): + replay(session, {'exitcode': 0, 'finished': finished, 'reports': []}, ['a', 'b'], parallel=True) + + +@pytest.mark.parametrize('parallel', [False, True]) +def test_finished_tests_must_be_strings(session: pytest.Session, parallel: bool) -> None: + """Require string completion identifiers in both serial and parallel results.""" + with pytest.raises(WorkerError, match='malformed finished'): + replay(session, {'exitcode': 0, 'finished': [{}], 'reports': []}, ['a'], parallel=parallel) + + +def test_parallel_failfast_can_finish_a_nonprefix_subset(session: pytest.Session) -> None: + """Allow parallel failfast to finish a later assignment before earlier tests run. + + Only the second assigned test reports a failure, demonstrating why a valid + parallel failfast result need not be a prefix of the assignment list. + """ + session.config.option.maxfail = 1 + reports = [xdist_report('b', 'setup'), xdist_report('b', 'call', outcome='failed'), xdist_report('b', 'teardown')] + replay(session, {'exitcode': 1, 'finished': ['b'], 'reports': reports}, ['a', 'b'], parallel=True) + assert session.testsfailed == 1 + + +@pytest.mark.parametrize(('reports', 'finished', 'message'), [ + ([xdist_report('a', 'setup')], ['a'], 'incomplete set'), + ([dict(xdist_report('a', 'teardown'), worker_id=None)], ['a'], 'worker ID'), + ([xdist_report('b', 'setup'), xdist_report('a', 'teardown')], ['a'], 'incomplete test phases'), + ([xdist_report('a', 'setup'), xdist_report('a', 'teardown')], ['a'], 'incomplete test phases'), + ([xdist_report('a', 'setup', 'gw1'), xdist_report('a', '???', outcome='failed')], ['a'], 'unterminated'), + ([xdist_report('a', '???')], ['a'], 'malformed crash phases'), + ([xdist_report('a', 'call'), xdist_report('a', '???', outcome='failed')], ['a'], 'malformed crash phases'), + ([xdist_report('a', 'teardown'), xdist_report('a', 'teardown')], ['a'], 'incomplete set'), +]) +def test_invalid_parallel_phases_fail(session: pytest.Session, reports: List[Dict[str, object]], finished: List[str], message: str) -> None: + """Reject inconsistent parallel report phases even when failfast permits partial results. + + Handcrafted report streams exercise invalid worker IDs, phase sequences, + and xdist's '???' crash phase without relying on nondeterministic crashes. + """ + session.config.option.maxfail = 1 + with pytest.raises(WorkerError, match=message): + replay(session, {'exitcode': 1, 'finished': finished, 'reports': reports}, ['a', 'b'], parallel=True) diff --git a/tests/test_settings.py b/tests/test_settings.py new file mode 100644 index 0000000..9c2be3d --- /dev/null +++ b/tests/test_settings.py @@ -0,0 +1,124 @@ +import json +import sys + +import pytest + +from throngtest.settings import read_settings + + +def test_defaults(pytester: pytest.Pytester) -> None: + """Use the documented defaults when no configuration source supplies values.""" + settings = read_settings(pytester.parseconfig()) + assert settings.workers == 4 + assert settings.check_fingerprints is False + assert settings.backend == 'temporary_directory' + assert settings.distribution == 'tests' + assert settings.python == sys.executable + assert '.git/' in settings.exclude + assert settings.preparation == [] + + +def test_precedence_and_fresh_environment(pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch) -> None: + """Resolve CLI over environment over TOML and reread changing environment values. + + Reusing the same pytest config after environment changes detects settings + cached between reads; explicit CLI values then verify the highest priority. + """ + pytester.makepyprojecttoml('[tool.throngtest]\nworkers = 2\nbackend = "local"\ndistribution = "files"\nexclude = ["large/"]') + config = pytester.parseconfig() + assert read_settings(config).workers == 2 + assert read_settings(config).backend == 'local' + assert read_settings(config).distribution == 'files' + assert read_settings(config).exclude == ['large/'] + monkeypatch.setenv('THRONGTEST_WORKERS', '3') + assert read_settings(config).workers == 3 + monkeypatch.setenv('THRONGTEST_WORKERS', '4') + assert read_settings(config).workers == 4 + config = pytester.parseconfig('--isolates=0', '--throngtest-distribution=tests', '--throngtest-python=custom python', '--throngtest-exclude=["custom/"]') + settings = read_settings(config) + assert settings.workers == 0 + assert settings.distribution == 'tests' + assert settings.python == 'custom python' + assert settings.exclude == ['custom/'] + + +@pytest.mark.parametrize(('name', 'value'), [('workers', '-1'), ('workers', 'nope'), ('backend', ' '), ('distribution', 'other'), ('python', ''), ('exclude', 'oops'), ('preparation', 'oops')]) +def test_invalid_cli(pytester: pytest.Pytester, name: str, value: str) -> None: + """Report invalid CLI values as plugin-specific pytest usage errors.""" + option = '--isolates' if name == 'workers' else f'--throngtest-{name}' + with pytest.raises(pytest.UsageError, match='throngtest:'): + read_settings(pytester.parseconfig(f'{option}={value}')) + + +def test_workers_cli_was_renamed(pytester: pytest.Pytester) -> None: + """Advertise --isolates in help and reject the removed --throngtest-workers option.""" + result = pytester.runpytest_subprocess('--help') + assert result.ret == pytest.ExitCode.OK + assert '--isolates=COUNT' in result.stdout.str() + assert '--throngtest-workers' not in result.stdout.str() + with pytest.raises(pytest.UsageError, match='unrecognized arguments: --throngtest-workers=2'): + pytester.parseconfig('--throngtest-workers=2') + + +def test_invalid_toml_type(pytester: pytest.Pytester) -> None: + """Reject a TOML worker count whose type does not match the setting.""" + pytester.makepyprojecttoml('[tool.throngtest]\nworkers = "two"') + with pytest.raises(pytest.UsageError, match='throngtest:'): + read_settings(pytester.parseconfig()) + + +def test_preparation_precedence_and_empty_override(pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch) -> None: + """Apply source precedence to preparation commands and allow explicit empty overrides.""" + pytester.makepyprojecttoml('[tool.throngtest]\npreparation = ["echo first", "echo second"]') + config = pytester.parseconfig() + assert read_settings(config).preparation == ['echo first', 'echo second'] + monkeypatch.setenv('THRONGTEST_PREPARATION', '["echo environment"]') + assert read_settings(config).preparation == ['echo environment'] + config = pytester.parseconfig('--throngtest-preparation=["echo CLI"]') + assert read_settings(config).preparation == ['echo CLI'] + config = pytester.parseconfig('--throngtest-preparation=[]') + assert read_settings(config).preparation == [] + monkeypatch.setenv('THRONGTEST_PREPARATION', '[]') + assert read_settings(pytester.parseconfig()).preparation == [] + + +@pytest.mark.parametrize('source', ['cli', 'environment', 'toml']) +@pytest.mark.parametrize('value', ['echo command', [42], [''], [' \t'], [['echo nested']]]) +def test_invalid_preparation(pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch, source: str, value: object) -> None: + """Reject malformed command lists and blank commands from every settings source.""" + encoded = json.dumps(value) + arguments = [] + if source == 'cli': + arguments = ['--throngtest-preparation=' + encoded] + elif source == 'environment': + monkeypatch.setenv('THRONGTEST_PREPARATION', encoded) + else: + pytester.makepyprojecttoml('[tool.throngtest]\npreparation = ' + encoded) + with pytest.raises(pytest.UsageError, match='throngtest:'): + read_settings(pytester.parseconfig(*arguments)) + + +def test_fingerprint_flag_precedence(pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch) -> None: + """Resolve fingerprint booleans by source priority and the last explicit CLI flag.""" + pytester.makepyprojecttoml('[tool.throngtest]\ncheck_fingerprints = true') + config = pytester.parseconfig() + assert read_settings(config).check_fingerprints is True + monkeypatch.setenv('THRONGTEST_CHECK_FINGERPRINTS', 'false') + assert read_settings(config).check_fingerprints is False + assert read_settings(pytester.parseconfig('--throngtest-check-fingerprints')).check_fingerprints is True + monkeypatch.setenv('THRONGTEST_CHECK_FINGERPRINTS', 'true') + assert read_settings(config).check_fingerprints is True + assert read_settings(pytester.parseconfig('--throngtest-no-check-fingerprints')).check_fingerprints is False + assert read_settings(pytester.parseconfig('--throngtest-check-fingerprints', '--throngtest-no-check-fingerprints')).check_fingerprints is False + assert read_settings(pytester.parseconfig('--throngtest-no-check-fingerprints', '--throngtest-check-fingerprints')).check_fingerprints is True + + +@pytest.mark.parametrize(('source', 'value'), [('environment', 'maybe'), ('environment', '[]'), ('toml', '1'), ('toml', '"true"'), ('toml', '[]')]) +def test_invalid_fingerprint_boolean(pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch, source: str, value: str) -> None: + """Reject invalid environment booleans and nonboolean TOML fingerprint settings.""" + if source == 'environment': + monkeypatch.setenv('THRONGTEST_CHECK_FINGERPRINTS', value) + else: + pytester.makepyprojecttoml('[tool.throngtest]\ncheck_fingerprints = ' + value) + with pytest.raises(pytest.UsageError, match='throngtest:'): + read_settings(pytester.parseconfig()) diff --git a/tests/test_xdist.py b/tests/test_xdist.py new file mode 100644 index 0000000..4f323d3 --- /dev/null +++ b/tests/test_xdist.py @@ -0,0 +1,403 @@ +import json +import shlex +import sys +from pathlib import Path +from xml.etree import ElementTree + +import pytest + +from tests import test_integration + + +@pytest.mark.parametrize('mode', ['load', 'loadfile', 'loadscope', 'loadgroup', 'worksteal']) +@pytest.mark.parametrize('checked', [False, True]) +def test_nested_workers_execute_only_their_shard(pytester: pytest.Pytester, backend: str, tmp_path: Path, mode: str, checked: bool) -> None: + """Distribute each isolate's shard across its own xdist workers without repeated tests. + + Exclusive external records identify every test's PID, shard, and worker, + proving that two isolates each use two workers. Tracing real backend calls + checks that preparation and pytest each start once per isolate; temporary + directory records also verify cleanup across schedulers and fingerprint modes. + """ + pytester.makeconftest(f''' + from pathlib import Path + from uuid import uuid4 + from throng.extensions.local.isolate import LocalIsolate + from throng.extensions.temporary_directory.isolate import TemporaryDirectoryIsolate + def observe(cls): + original = cls.run + def traced(self, command, *args, **kwargs): + (Path({str(tmp_path)!r}) / ('dispatch-' + uuid4().hex)).write_text(command) + return original(self, command, *args, **kwargs) + cls.run = traced + observe(LocalIsolate) + observe(TemporaryDirectoryIsolate) + ''') + source = f''' + import json, os + from pathlib import Path + import pytest + from throngtest.protocol import Request + @pytest.mark.parametrize('index', range(4)) + def test_item(index, worker_id, request): + assert worker_id != 'master' + assert Path('prepared').read_text() == 'ready' + shard = Request.unpack(request.config.workerinput['throngtest']).shard + record = Path({str(tmp_path)!r}) / (Path(__file__).stem + '-' + str(index) + '.json') + with record.open('x') as stream: + json.dump({{'pid': os.getpid(), 'shard': shard, 'worker': worker_id, 'cwd': str(Path.cwd())}}, stream) + ''' + pytester.makepyfile(**{f'test_{index}': source for index in range(4)}) + preparation = shlex.join([sys.executable, '-c', "from pathlib import Path; Path('prepared').write_text('ready')"]) + result = pytester.runpytest_subprocess( + '--isolates=2', '-n', '2', f'--dist={mode}', '--throngtest-distribution=files', f'--throngtest-backend={backend}', + '--throngtest-preparation=' + json.dumps([preparation]), *(['--throngtest-check-fingerprints'] if checked else []), timeout=60, + ) + result.assert_outcomes(passed=16) + records = [json.loads(record.read_text()) for record in tmp_path.glob('*.json')] + assert len(records) == 16 + assert len({record['pid'] for record in records}) == 4 + assert {record['shard'] for record in records} == {0, 1} + assert len(list(tmp_path.glob('dispatch-*'))) == 4 # One preparation and one pytest command per isolate. + for shard in (0, 1): + assert {record['worker'] for record in records if record['shard'] == shard} == {'gw0', 'gw1'} + if backend == 'temporary_directory': + directories = {record['cwd'] for record in records} + assert len(directories) == 2 + assert all(not Path(directory).exists() for directory in directories) + assert not (pytester.path / 'prepared').exists() + + +def test_plugin_works_without_xdist(pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch) -> None: + """Run throngtest without loading or importing xdist. + + Disabling plugin autoload and explicitly enabling throngtest exercises this + path even though xdist is installed in the test environment. + """ + monkeypatch.setenv('PYTEST_DISABLE_PLUGIN_AUTOLOAD', '1') + pytester.makepyfile(''' + import sys + def test_no_xdist(request): + assert not request.config.pluginmanager.hasplugin('xdist') + assert 'xdist' not in sys.modules + ''') + pytester.runpytest_subprocess('-p', 'throngtest.plugin', '--isolates=2').assert_outcomes(passed=1) + + +@pytest.mark.parametrize('arguments', [(), ('-n', '0')]) +def test_installed_xdist_stays_inactive(pytester: pytest.Pytester, arguments: tuple) -> None: + """Keep xdist inactive inside isolates unless its worker count is explicitly enabled.""" + pytester.makepyfile(''' + def test_no_workers(worker_id): + assert worker_id == 'master' + ''') + pytester.runpytest_subprocess('--isolates=1', *arguments).assert_outcomes(passed=1) + + +def test_zero_isolates_preserves_standalone_xdist(pytester: pytest.Pytester) -> None: + """Let xdist operate independently when --isolates=0 disables throngtest execution.""" + pytester.makepyfile(''' + def test_xdist(worker_id, request): + assert worker_id != 'master' + assert 'throngtest' not in request.config.workerinput + assert not request.config.pluginmanager.hasplugin('throngtest-runner') + ''') + pytester.runpytest_subprocess('--isolates=0', '-n', '2').assert_outcomes(passed=1) + + +@pytest.mark.parametrize('scenario', ['test_reports_and_junit', 'test_selection_and_conftest', 'test_collect_only_does_not_execute', 'test_empty_collection', 'test_collection_error']) +def test_nested_regressions(pytester: pytest.Pytester, backend: str, monkeypatch: pytest.MonkeyPatch, scenario: str) -> None: + """Preserve existing pytest integration behavior with xdist enabled inside isolates. + + The test calls shared integration scenarios directly after injecting -n2 + through PYTEST_ADDOPTS, reusing their assertions for reports, selection, + collect-only, empty projects, and collection failures. + """ + monkeypatch.setenv('PYTEST_ADDOPTS', '-n2') + function = getattr(test_integration, scenario) + function(pytester, backend) + + +@pytest.mark.parametrize('checked', [False, True]) +def test_nested_absolute_parameter_paths(pytester: pytest.Pytester, checked: bool) -> None: + """Respect fingerprint opt-in when nested workers collect cwd-dependent parameter IDs. + + The parameter contains an absolute path that changes in the temporary + isolate, while xdist workers within that isolate see the same path. + """ + pytester.makepyfile(''' + from pathlib import Path + import pytest + @pytest.mark.parametrize('path', [str(Path.cwd() / 'missing')]) + def test_path(path): assert Path(path).parent == Path.cwd() + ''') + args = ['--throngtest-check-fingerprints'] if checked else [] + result = pytester.runpytest_subprocess('-n2', *args, timeout=45) + if checked: + assert result.ret == pytest.ExitCode.INTERNAL_ERROR + assert 'Possible causes to check:' in result.stdout.str() + else: + result.assert_outcomes(passed=1) + + +@pytest.mark.parametrize('checked', [False, True]) +def test_nested_duplicate_nodeids(pytester: pytest.Pytester, backend: str, checked: bool) -> None: + """Preserve duplicate collected occurrences when xdist schedules an isolate's tests.""" + test = pytester.makepyfile('def test_ok(): pass') + args = ['--throngtest-check-fingerprints'] if checked else [] + pytester.runpytest_subprocess('--isolates=1', '-n2', f'--throngtest-backend={backend}', *args, str(test), str(test), timeout=45).assert_outcomes(passed=2) + + +@pytest.mark.parametrize('source', ['cli', 'environment', 'ini']) +def test_user_configures_xdist_and_auto_is_resolved_inside_each_isolate(pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch, source: str) -> None: + """Honor native xdist option sources and resolve auto worker counts inside isolates. + + A custom auto-count hook asserts the isolate worker marker before returning + two. This detects premature resolution in the outer controller, while + workerinput verifies the count received by the resulting xdist processes. + """ + pytester.makeconftest(''' + from throngtest.settings import WORKER + def pytest_xdist_auto_num_workers(config): + assert config.stash.get(WORKER, False), 'xdist must not start in the outer controller' + return 2 + ''') + pytester.makepyfile(''' + import pytest + @pytest.mark.parametrize('index', range(8)) + def test_item(index, request, worker_id): + assert worker_id != 'master' + assert request.config.workerinput['workercount'] == 2 + ''') + args = ['--numprocesses=auto'] + if source == 'environment': + monkeypatch.setenv('PYTEST_ADDOPTS', '--numprocesses=auto') + args = [] + elif source == 'ini': + pytester.makeini('[pytest]\naddopts = --numprocesses=auto') + args = [] + pytester.runpytest_subprocess('--isolates=2', *args, timeout=45).assert_outcomes(passed=8) + + +def test_xdist_loadgroup_marks_and_file_distribution(pytester: pytest.Pytester, backend: str, tmp_path: Path) -> None: + """Keep marked tests on one xdist worker with file distribution and fingerprint checks. + + Multiple group marks and a name containing '@' exercise xdist's group ID + handling. External PID records verify that every case runs on one process. + """ + pytester.makepyfile(f''' + import os + from pathlib import Path + import pytest + @pytest.mark.xdist_group('first@group') + @pytest.mark.xdist_group('second') + @pytest.mark.parametrize('index', range(8)) + def test_item(index): + (Path({str(tmp_path)!r}) / str(index)).write_text(str(os.getpid())) + ''') + result = pytester.runpytest_subprocess('--isolates=2', '-n2', '--dist=loadgroup', '--throngtest-check-fingerprints', f'--throngtest-backend={backend}', '--throngtest-distribution=files', timeout=45) + result.assert_outcomes(passed=8) + assert len({path.read_text() for path in tmp_path.iterdir()}) == 1 + + +def test_nested_failfast(pytester: pytest.Pytester, backend: str) -> None: + """Accept early xdist termination under -x without treating unfinished work as corruption. + + Either of two workers may report an in-flight failure before stopping, so + the expected failure count allows both outcomes of that scheduling race. + """ + pytester.makepyfile(''' + import pytest + @pytest.mark.parametrize('index', range(20)) + def test_fail(index): assert False + ''') + result = pytester.runpytest_subprocess('--isolates=1', '-n2', '-x', f'--throngtest-backend={backend}', timeout=45) + assert result.ret == pytest.ExitCode.TESTS_FAILED + assert 1 <= result.parseoutcomes()['failed'] <= 2 + assert 'exactly once' not in result.stdout.str() + + +@pytest.mark.parametrize('restart', ['0', '1']) +def test_nested_worker_crash(pytester: pytest.Pytester, backend: str, restart: str) -> None: + """Report an xdist worker crash and honor the configured restart policy. + + os._exit kills the process without normal pytest teardown. With a restart + available, the remaining tests must run; both policies must produce one + crash failure and one JUnit error in the outer controller. + """ + pytester.makepyfile(''' + import os + def test_crash(): os._exit(17) + def test_ok(): pass + def test_also_ok(): pass + ''') + result = pytester.runpytest_subprocess('--isolates=1', '-n1', '--max-worker-restart=' + restart, f'--throngtest-backend={backend}', '--junitxml=results.xml', timeout=45) + assert result.ret == pytest.ExitCode.TESTS_FAILED + assert result.parseoutcomes()['failed'] == 1 + assert 'crashed while running' in result.stdout.str() + if restart == '1': + result.assert_outcomes(passed=2, failed=1) + assert len(ElementTree.parse(pytester.path / 'results.xml').findall('.//error')) == 1 + + +@pytest.mark.parametrize('arguments', [('-n2', '--dist=each'), ('--dist=load', '--tx=popen'), ('-n2', '--px=id=proxy//popen'), ('--looponfail',)]) +def test_unsupported_nested_modes_have_clear_errors(pytester: pytest.Pytester, arguments: tuple) -> None: + """Reject repetition, remote-worker, and loop-on-failure modes in nested execution. + + Older xdist versions do not expose --px, so that case accepts pytest's own + unknown-option diagnostic instead of requiring a throngtest validation error. + """ + pytester.makepyfile('def test_never(): assert False') + result = pytester.runpytest_subprocess('--isolates=2', *arguments) + assert result.ret == pytest.ExitCode.USAGE_ERROR + if any(argument.startswith('--px=') for argument in arguments) and not hasattr(pytester.parseconfig().option, 'px'): + assert 'unrecognized arguments: --px' in result.stderr.str() + else: + assert 'throngtest:' in result.stderr.str() + + +def test_auto_can_disable_xdist_inside_isolate(pytester: pytest.Pytester) -> None: + """Run tests without xdist workers when the isolate's auto-count hook returns zero.""" + pytester.makeconftest('def pytest_xdist_auto_num_workers(config): return 0') + pytester.makepyfile('def test_ok(worker_id): assert worker_id == "master"') + pytester.runpytest_subprocess('--isolates=1', '-n', 'auto', timeout=45).assert_outcomes(passed=1) + + +def test_nested_xdist_still_checks_its_own_collections(pytester: pytest.Pytester) -> None: + """Retain xdist's collection consistency checks when throngtest fingerprints are disabled. + + Appending each native worker ID to collected node IDs creates a disagreement + within one isolate, which xdist must detect before any test body runs. + """ + pytester.makeconftest(''' + def pytest_collection_modifyitems(config, items): + if hasattr(config, 'workerinput'): + for item in items: + item._nodeid += config.workerinput['workerid'] + ''') + pytester.makepyfile('def test_never(): assert False, "must not run"') + result = pytester.runpytest_subprocess('--isolates=1', '-n2', timeout=45) + assert result.ret == pytest.ExitCode.INTERRUPTED + output = result.stdout.str() + result.stderr.str() + assert 'Different tests were collected between' in output + assert 'gw0' in output + assert 'gw1' in output + + +@pytest.mark.parametrize('count', [0, 1]) +def test_nested_empty_shards(pytester: pytest.Pytester, backend: str, count: int) -> None: + """Handle empty nested shards and preserve the exit code when every shard is empty. + + The controller collects four cases, but a worker-only hook retains zero or + one. This creates empty assignments after isolates have already been selected. + """ + pytester.makeconftest(f''' + from throngtest.settings import WORKER + def pytest_collection_modifyitems(config, items): + if config.stash.get(WORKER, False): + items[:] = items[:{count}] + ''') + pytester.makepyfile(''' + import pytest + @pytest.mark.parametrize('index', range(4)) + def test_item(index): pass + ''') + result = pytester.runpytest_subprocess('--isolates=2', '-n2', f'--throngtest-backend={backend}', timeout=45) + result.assert_outcomes(passed=count) + assert result.ret == (pytest.ExitCode.OK if count else pytest.ExitCode.NO_TESTS_COLLECTED) + + +def test_nested_warnings(pytester: pytest.Pytester, backend: str) -> None: + """Forward an xdist worker warning exactly once through the isolate controller.""" + pytester.makepyfile(''' + import warnings + def test_warning(): warnings.warn('nested warning', UserWarning) + ''') + result = pytester.runpytest_subprocess('--isolates=1', '-n2', f'--throngtest-backend={backend}', timeout=45) + result.assert_outcomes(passed=1, warnings=1) + assert 'nested warning' in result.stdout.str() + + +@pytest.mark.parametrize('stage', ['setup', 'call', 'teardown']) +def test_grouped_worker_crash_without_restart(pytester: pytest.Pytester, stage: str) -> None: + """Report a grouped test crashing during setup, call, or teardown without restarting xdist. + + The generated fixture or test exits abruptly at the selected phase. With + restarts disabled, the controller must already know the grouped test's + identity, without relying on worker shutdown output or a replacement process. + """ + pytester.makepyfile(f''' + import os + import pytest + @pytest.fixture + def resource(): + if {stage!r} == 'setup': os._exit(17) + yield + if {stage!r} == 'teardown': os._exit(17) + @pytest.mark.xdist_group('group') + def test_crash(resource): + if {stage!r} == 'call': os._exit(17) + ''') + result = pytester.runpytest_subprocess('--isolates=1', '-n1', '--dist=loadgroup', '--max-worker-restart=0', '--throngtest-check-fingerprints', timeout=45) + assert result.ret == pytest.ExitCode.TESTS_FAILED + assert result.parseoutcomes()['failed'] == 1 + assert 'crashed while running' in result.stdout.str() + + +def test_preparation_failure_prevents_xdist_start(pytester: pytest.Pytester, tmp_path: Path) -> None: + """Fail preparation before starting any nested xdist workers. + + A pytest_configure_node hook would create an external marker on worker + startup, so its absence checks the startup boundary as well as the exit code. + """ + pytester.makeconftest(f''' + from pathlib import Path + def pytest_configure_node(node): + Path({str(tmp_path / 'started')!r}).touch() + ''') + pytester.makepyfile('def test_never(): assert False') + commands = [shlex.join([sys.executable, '-c', 'raise SystemExit(7)'])] + result = pytester.runpytest_subprocess('--isolates=2', '-n2', '--throngtest-preparation=' + json.dumps(commands), timeout=45) + assert result.ret == pytest.ExitCode.INTERNAL_ERROR + assert 'preparation command 1 failed with exit code 7' in result.stdout.str() + assert not (tmp_path / 'started').exists() + + +def test_nested_run_from_subdirectory_with_absolute_nodeid(pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch) -> None: + """Preserve an absolute node selector when nested execution starts from a subdirectory.""" + pytester.makeini('[pytest]') + child = pytester.path / 'child' + child.mkdir() + test = child / 'test_example.py' + test.write_text('def test_ok(): pass\ndef test_bad(): assert False\n') + monkeypatch.chdir(child) + pytester.runpytest_subprocess('--isolates=2', '-n2', str(test) + '::test_ok', timeout=45).assert_outcomes(passed=1) + + +@pytest.mark.parametrize('order', ['tryfirst=True', 'trylast=True', '']) +@pytest.mark.parametrize('checked', [False, True]) +def test_loadgroup_preserves_other_plugins_nodeid_changes(pytester: pytest.Pytester, checked: bool, order: str) -> None: + """Preserve third-party node ID changes around xdist's loadgroup suffix handling. + + The custom collection hook runs with first, last, or default priority. + JUnit must retain its '[custom]' suffix while omitting xdist's group suffix, + including when fingerprints compare the controller and worker collections. + """ + pytester.makeconftest(f''' + import pytest + @pytest.hookimpl({order}) + def pytest_collection_modifyitems(items): + for item in items: + item._nodeid += '[custom]' + ''') + pytester.makepyfile(''' + import pytest + @pytest.mark.xdist_group('group') + def test_ok(): pass + ''') + args = ['--throngtest-check-fingerprints'] if checked else [] + result = pytester.runpytest_subprocess('--isolates=1', '-n2', '--dist=loadgroup', '--junitxml=results.xml', *args, timeout=45) + result.assert_outcomes(passed=1) + case = ElementTree.parse(pytester.path / 'results.xml').find('.//testcase') + assert case is not None + assert case.attrib['name'] == 'test_ok[custom]' From 183a4dda0f0f8afb22ac3b85b8f866a2ece4a4a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=95=D0=B2=D0=B3=D0=B5=D0=BD=D0=B8=D0=B9=20=D0=91=D0=BB?= =?UTF-8?q?=D0=B8=D0=BD=D0=BE=D0=B2?= Date: Tue, 29 Sep 2026 16:04:27 +0300 Subject: [PATCH 10/27] Write comprehensive throngtest README --- README.md | 239 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 238 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a5f2206..464a756 100644 --- a/README.md +++ b/README.md @@ -1 +1,238 @@ -# pytest-throng \ No newline at end of file +# throngtest + +Run pytest test subsets in [throng](https://github.com/mutating/throng) isolates. +Throngtest is an independent pytest plugin: it has its own options and does not +depend on pytest-xdist or implement xdist's flags or fixtures. + +```bash +pip install throngtest +pytest +pytest --throngtest-distribution=files +pytest --isolates=2 --throngtest-backend=local +``` + +Installing the plugin enables distribution into up to four isolates by default. +Use `--isolates=0` to disable it and run ordinary pytest. +Python 3.8+ and pytest 8.3.5–9.x are supported. + +## Configuration + +All throngtest settings are loaded, converted and validated through +[skelet](https://github.com/mutating/skelet). Sources have this precedence: + +1. Explicit `--throngtest-*` arguments, including arguments supplied by pytest's + `addopts` or `PYTEST_ADDOPTS`. +2. Environment variables with the `THRONGTEST_` prefix. +3. `[tool.throngtest]` in `pyproject.toml` at pytest's `rootdir`. +4. Defaults. + +```toml +[tool.throngtest] +workers = 4 +check_fingerprints = false +backend = "temporary_directory" +distribution = "files" +preparation = ["python scripts/prepare.py", "python scripts/seed_test_data.py"] +``` + +```bash +THRONGTEST_WORKERS=4 pytest +pytest --isolates=2 --throngtest-exclude='[".git/", ".venv/", "large-data/"]' +``` + +| Setting | CLI option | Default | Meaning | +| --- | --- | --- | --- | +| `workers` | `--isolates` | `4` | Maximum number of nonempty test subsets; a nonnegative integer. `0` disables distribution. | +| `check_fingerprints` | `--throngtest-check-fingerprints` | `false` | Require identical ordered collections in the controller and isolates. | +| `backend` | `--throngtest-backend` | `temporary_directory` | Name of an installed throng plugin. | +| `distribution` | `--throngtest-distribution` | `tests` | Split individual tests or keep each file together (`files`). | +| `python` | `--throngtest-python` | Controller's `sys.executable` | Python executable available inside each isolate. | +| `exclude` | `--throngtest-exclude` | See below | Throng snapshot exclusion patterns; a JSON array for CLI/environment sources and an array in TOML. | +| `preparation` | `--throngtest-preparation` | `[]` | Ordered list of nonempty commands run once in each isolate before pytest; JSON for CLI/environment sources and an array in TOML. | + +The default exclusions are `.git/`, `.venv/`, `venv/`, `__pycache__/`, +`.pytest_cache/`, `.mypy_cache/`, `.ruff_cache/`, `build/`, `dist/`, and `mutants/`. +An explicit exclusion list replaces the defaults. Patterns are interpreted by +throng. Configurations are read afresh for each pytest session. + +Fingerprint checks are disabled by default. Enable them with +`--throngtest-check-fingerprints`, `THRONGTEST_CHECK_FINGERPRINTS=true`, or +`check_fingerprints = true` in `[tool.throngtest]`. Use +`--throngtest-no-check-fingerprints` to override an enabled setting from the +environment or TOML. Both CLI flags take no value; if both are supplied, the +last flag wins. The environment accepts `true`/`false`; TOML uses booleans. + +## Isolate preparation + +Use `preparation` to generate files, install dependencies, or otherwise prepare +each isolate before its pytest process starts: + +```bash +pytest --throngtest-preparation='["python scripts/prepare.py"]' +THRONGTEST_PREPARATION='["python scripts/prepare.py"]' pytest +``` + +Commands run in the listed order through `isolate.run`, in the same isolate as +the tests. With the built-in backends they start at the project root, even when +pytest is invoked from a subdirectory. Each call starts a separate process: +file changes persist, but `cd`, `export`, and shell activation do not carry over +to later commands or pytest. To select a prepared interpreter, set `python` to +its executable path. Command syntax follows the chosen throng backend. + +A nonzero command exit code stops preparation of that isolate and prevents its +tests from starting. The controller cancels outstanding work, cleans up the +isolates, and exits with code 3, reporting the failed command, its exit code, +and preparation output. Other isolates may already have started their tests. +Successful preparation output is forwarded with the isolate's test results. + +An explicit list replaces the lower-priority list; `[]` disables preparation. +No preparation runs with `workers = 0`, `--collect-only`, or an empty test +selection. The controller collects tests before creating isolates, so its +environment must already support that initial collection. Preparation runs +before collection inside each worker, not before controller collection. + +## Distribution and execution + +The controller collects and selects tests using pytest. It partitions the +result into at most `workers` nonempty subsets. In `tests` mode, tests are +assigned in round-robin order. In `files` mode, larger files are assigned first +to the least populated subset, using test counts as the size estimate. Original +collection order is preserved within each subset. Durations are not predicted +and work is not reassigned between subsets. + +Each subset gets one isolate from a single throng manager. A service thread +waits for its synchronous command; test functions themselves execute in pytest +inside the isolate. Every worker independently collects and partitions tests. +The number of partitions is limited to the number of isolates actually started. +With `check_fingerprints = false` (the default), the collections are not compared +with the controller, and reports retain the identifiers collected inside each +isolate. Report completeness is still validated against that isolate's assigned +subset, including fail-fast handling. Empty subsets are allowed; if no isolate +executes any tests, the overall run returns pytest's exit code 5. + +Without fingerprint checks, differences in collection order or contents between +isolates can cause tests to be omitted or executed more than once. Enable +`check_fingerprints` when you need to reject these differences before execution. +With checks enabled, each worker compares its ordered collection fingerprint +before running tests. A collection mismatch first lists possible causes to check: +file changes or exclusions, preparation, environment-dependent parametrization, +unstable ordering, different selection settings or plugins, and absolute paths +in parameter IDs that change when the project is copied. These are diagnostic +suggestions, not an automatic determination of the cause. It then reports both +selected-test counts and a unified diff of the ordered test identifiers (`-` for the controller, `+` for +the isolate). The diff includes parameter IDs, preserves duplicates, and is +limited to 100 lines with an explicit truncation notice for larger differences. + +The built-in backends have different guarantees: + +* `temporary_directory` copies the project into a separate temporary directory + for each isolate and removes it after execution. Commands in different + isolates can run concurrently. +* `local` executes in the original project directory. Its isolates use separate + command processes, but throng 0.0.3 serializes commands belonging to one local + manager. Project file changes are visible to other subsets and remain after + the run. + +Throng determines isolation and available concurrency. Throngtest does not +bypass a manager's synchronization. Without nested xdist, a session fixture runs once **per isolate**; +module and class fixtures may also be instantiated in multiple isolates in +`tests` mode. Use `files` to keep a file's fixtures and tests together. + +## Optional cooperation with pytest-xdist + +Throngtest does not depend on, install, or enable pytest-xdist. Users who want +both levels of distribution install and configure xdist separately: + +```bash +python -m pip install pytest-xdist +pytest --isolates=2 -n 4 --dist=load +``` + +Throngtest partitions the tests between up to two isolates. Each isolate runs +its preparation once, then starts its own xdist controller with four local +workers. Xdist distributes only that isolate's subset. The example can execute +up to eight tests concurrently with `temporary_directory`; the `local` backend +still serializes isolate commands. Processes belonging to the same isolate +share its prepared filesystem. A session fixture runs once per xdist worker. + +`-n`, `--numprocesses`, `--dist`, and other xdist options belong to xdist and +retain its parsing and configuration through CLI, pytest `addopts`, and +`PYTEST_ADDOPTS`. Throngtest adds no xdist settings or aliases. Without an +enabled xdist run, execution stays sequential within each isolate. `-n 0` +disables inner distribution; `--isolates=0` leaves standalone xdist in control. +`-n auto` is resolved independently in each isolate and is not a global process +budget. Xdist's `worker_id` values, such as `gw0`, are local to each isolate. + +The supported schedulers are `load`, `loadfile`, `loadscope`, `loadgroup`, and +`worksteal`. Their grouping guarantees apply within each assigned subset. +For example, use `--throngtest-distribution=files --dist=loadfile` to keep a +file together at both levels. Xdist groups do not combine tests from different +isolates. `--dist=each` is rejected because it intentionally repeats tests. +Explicit `--tx`/`--px` execution environments and `--looponfail` are unsupported; +inner workers must be local to the isolate and configured with `-n`. + +Xdist still checks collection consistency between its own workers even when +throngtest's `check_fingerprints` is disabled. Throngtest accepts interleaved +results and validates completed executions, including duplicate node IDs, +fail-fast stops, and xdist worker crashes. Xdist's worker restart settings are +respected, and crash reports reach the combined terminal and JUnit output. +Xdist's usual limitations also apply, including its lack of live stdout +forwarding with `-s`. Captured output on failures is retained. + +## Results and failures + +Workers return JSON reports through the command's stdout. No shared report +directory, network listener, or pickle transport is required. Third-party +throng backends must provide stdout and a process return code. The selected +interpreter must have throngtest, pytest, the project's dependencies, and +required pytest plugins available by the end of preparation. Throngtest only +installs packages when explicitly instructed through preparation commands. + +The controller replays pytest reports, preserving assertion explanations, +captured output, skip/xfail/xpass results, setup/teardown failures, durations, +and test properties. `--junitxml` produces one aggregate report. Runtime warnings +are forwarded with their original category name in the message. `-s` output is +forwarded after its subset completes. Reports are buffered per subset; there is +no live per-test progress from a running isolate. + +`-k`, `-m`, explicit node IDs, parametrization, conftest fixtures, `--collect-only`, +and `--continue-on-collection-errors` are supported. Empty collections retain +pytest's exit code 5. Isolate process crashes, missing interpreters, collection mismatches +with fingerprint checks enabled, and incomplete report payloads fail with exit code 3. A reported +worker interrupt propagates exit code 2. + +`-x` and `--maxfail` apply within each worker and to the aggregate reports. Once +the controller observes the limit, it cancels outstanding commands through +throng's cancellation token and cleans up isolates. Other subsets may already +have executed additional tests. Cancellation responsiveness depends on the +backend; throng's isolate creation itself does not accept a cancellation token. + +The plugin rejects interactive `--pdb`, cache-based selection (`--lf`, `--ff`, `--nf`), and +`--stepwise`. Other plugins must be importable in each worker, for example via +installation, `-p`, or conftest; programmatically injected plugin objects cannot +be transported. Plugins that implement custom test protocols, reruns, or their +own output artifacts need separate compatibility work. The built-in throng +backends reuse the interpreter environment; temporary file copies are not +container or virtual-environment isolation. + +## Development + +```bash +python -m pip install -r requirements_dev.txt -e . +python -m pytest --isolates=0 +ruff check throngtest tests +mypy --strict --disallow-any-decorated --disallow-any-explicit \ + --disallow-any-expr --disallow-any-generics --disallow-any-unimported \ + --disallow-subclassing-any --warn-return-any throngtest +mypy tests +``` + +The outer test session disables distribution; integration tests launch their +own pytest sessions to exercise throngtest, including its default settings. +The test suite includes actual subprocess runs through both built-in throng +plugins, trace checks of their isolate APIs, complete/disjoint distribution, +concurrency barriers, configuration precedence, preparation, native reports, cancellation, +cleanup, and fault injection at the protocol/backend boundaries. The existing +CI also checks statement and branch coverage across Python and OS versions. +Subprocess coverage uses the startup hook configured in the CI workflow; tests +keep coverage data outside temporary isolate directories so it survives cleanup. From 8d1e88907f9fc1c2bc805cb2f3aa0e0c46f8794b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=95=D0=B2=D0=B3=D0=B5=D0=BD=D0=B8=D0=B9=20=D0=91=D0=BB?= =?UTF-8?q?=D0=B8=D0=BD=D0=BE=D0=B2?= Date: Tue, 29 Sep 2026 16:05:23 +0300 Subject: [PATCH 11/27] Update project URLs to point to throngtest repository --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c8bf40f..3c19500 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,5 +71,5 @@ lint.isort.combine-as-imports = true format.quote-style = "single" [project.urls] -'Source' = 'https://github.com/mutating/pytest-throng' -'Tracker' = 'https://github.com/mutating/pytest-throng/issues' +'Source' = 'https://github.com/mutating/throngtest' +'Tracker' = 'https://github.com/mutating/throngtest/issues' From b06157eafe824451811d06e7c7640bc576ea19e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=95=D0=B2=D0=B3=D0=B5=D0=BD=D0=B8=D0=B9=20=D0=91=D0=BB?= =?UTF-8?q?=D0=B8=D0=BD=D0=BE=D0=B2?= Date: Tue, 29 Sep 2026 16:22:52 +0300 Subject: [PATCH 12/27] Add coverage paths to merge installed and checkout packages --- pyproject.toml | 4 ++++ tests/test_coverage.py | 45 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 tests/test_coverage.py diff --git a/pyproject.toml b/pyproject.toml index 3c19500..8db0d30 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,6 +61,10 @@ parallel = true plugins = ["coverage_pyver_pragma"] source = ["throngtest"] +[tool.coverage.paths] +# Unit tests import the checkout; isolate subprocesses import the installed wheel. +source = ["throngtest", "*/site-packages/throngtest"] + [tool.pytest.ini_options] norecursedirs = ["build", "mutants"] diff --git a/tests/test_coverage.py b/tests/test_coverage.py new file mode 100644 index 0000000..4b9e1a1 --- /dev/null +++ b/tests/test_coverage.py @@ -0,0 +1,45 @@ +import subprocess +import sys +from pathlib import Path + +import pytest +from coverage import CoverageData + + +@pytest.mark.parametrize('branch', [False, True]) +@pytest.mark.parametrize('installed_path', [ + '/Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages/throngtest/distribution.py', + '/opt/hostedtoolcache/Python/3.10.11/x64/lib/python3.10/site-packages/throngtest/distribution.py', + r'C:\hostedtoolcache\windows\Python\3.10.11\x64\Lib\site-packages\throngtest\distribution.py', +]) +def test_coverage_combines_checkout_and_installed_package(tmp_path: Path, branch: bool, installed_path: str) -> None: + """Merge checkout and installed-package coverage into one file without losing hits. + + Synthetic data files give each package location different line or branch + hits, reproducing the split between unit tests and isolate subprocesses. + Real coverage combination uses the repository configuration and installed + paths from all three CI platforms, without requiring those paths to exist. + Combination runs in a child process because creating a Coverage instance + here would disable automatic coverage saving for the current pytest worker. + """ + root = Path(__file__).resolve().parents[1] + source = str(root / 'throngtest' / 'distribution.py') + basename = str(tmp_path / '.coverage') + for index, path in enumerate((source, installed_path)): + data = CoverageData(basename=basename, suffix=str(index)) + if branch: + data.add_arcs({path: [(10 + index, 11 + index)]}) + else: + data.add_lines({path: [10 + index, 11 + index]}) + data.write() + result = subprocess.run( + [sys.executable, '-m', 'coverage', 'combine', '--rcfile=' + str(root / 'pyproject.toml'), '--data-file=' + basename], + cwd=root, capture_output=True, text=True, check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + combined = CoverageData(basename=basename) + combined.read() + assert combined.measured_files() == {source} + assert set(combined.lines(source) or []) == {10, 11, 12} + if branch: + assert set(combined.arcs(source) or []) == {(10, 11), (11, 12)} From b0eef0ec4715a578ace650a2958762c5ed05b806 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=95=D0=B2=D0=B3=D0=B5=D0=BD=D0=B8=D0=B9=20=D0=91=D0=BB?= =?UTF-8?q?=D0=B8=D0=BD=D0=BE=D0=B2?= Date: Tue, 29 Sep 2026 16:34:23 +0300 Subject: [PATCH 13/27] Parametrize diagnostic test to cover Windows-style separators --- tests/test_integration.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tests/test_integration.py b/tests/test_integration.py index 0d79482..5f933f4 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -257,17 +257,23 @@ def test_item(value): raise AssertionError('test must not execute') assert 'THRONGTEST_' not in output -def test_collection_diagnostic_for_absolute_parameter_path(pytester: pytest.Pytester) -> None: +@pytest.mark.parametrize('backslash_separators', [False, True]) +def test_collection_diagnostic_for_absolute_parameter_path(pytester: pytest.Pytester, backslash_separators: bool) -> None: """Explain fingerprint mismatches caused solely by relocated absolute parameter paths. A parameter derives its ID from the current directory, so a temporary copy changes the identifier without changing the collection size. The diff must - expose both paths while omitting the encoded worker response. + expose both paths while omitting the encoded worker response. Pytest doubles + Windows backslashes in parameter IDs, so expectations use that same spelling. + Rewriting separators also exercises this behavior on non-Windows systems. """ - pytester.makepyfile(''' + path_expression = "str(Path.cwd() / 'missing-command')" + if backslash_separators: + path_expression += ".replace('/', chr(92))" + pytester.makepyfile(f''' from pathlib import Path import pytest - @pytest.mark.parametrize('value', [str(Path.cwd() / 'missing-command')]) + @pytest.mark.parametrize('value', [{path_expression}]) def test_item(value): raise AssertionError('test must not execute') ''') result = run(pytester, 'temporary_directory', '--throngtest-check-fingerprints') @@ -277,6 +283,9 @@ def test_item(value): raise AssertionError('test must not execute') assert output.index('Absolute paths in parameter IDs change') < output.index('controller: 1 selected tests') assert 'even if the tests are otherwise equivalent' in output expected = str(pytester.path.resolve() / 'missing-command') + if backslash_separators: + expected = expected.replace('/', '\\') + expected = expected.replace('\\', '\\\\') assert f'-test_collection_diagnostic_for_absolute_parameter_path.py::test_item[{expected}]' in output assert '+test_collection_diagnostic_for_absolute_parameter_path.py::test_item[' in output assert f'+test_collection_diagnostic_for_absolute_parameter_path.py::test_item[{expected}]' not in output From f695d6b1e446c59ac406c6d0feae72f2e2a65534 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=95=D0=B2=D0=B3=D0=B5=D0=BD=D0=B8=D0=B9=20=D0=91=D0=BB?= =?UTF-8?q?=D0=B8=D0=BD=D0=BE=D0=B2?= Date: Tue, 29 Sep 2026 16:59:09 +0300 Subject: [PATCH 14/27] Optimize interrupted status check in worker --- throngtest/worker.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/throngtest/worker.py b/throngtest/worker.py index 644bcff..a6c6616 100644 --- a/throngtest/worker.py +++ b/throngtest/worker.py @@ -131,7 +131,8 @@ def main(value: str) -> None: nodeid = cast(str, report['nodeid']) report['nodeid'] = worker.names.get(nodeid, nodeid) worker.finished = [worker.names.get(nodeid, nodeid) for nodeid in worker.finished] - if status == pytest.ExitCode.INTERRUPTED and worker.config is not None and cast(int, worker.config.getoption('maxfail')) and any(report['outcome'] == 'failed' for report in worker.reports): + outcomes = [report['outcome'] for report in worker.reports] + if status == pytest.ExitCode.INTERRUPTED and worker.config is not None and cast(int, worker.config.getoption('maxfail')) and 'failed' in outcomes: status = pytest.ExitCode.TESTS_FAILED response: Dict[str, object] = { 'version': 1, From e5f7591d850d9f31c066c65a0ed366e209e6fcfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=95=D0=B2=D0=B3=D0=B5=D0=BD=D0=B8=D0=B9=20=D0=91=D0=BB?= =?UTF-8?q?=D0=B8=D0=BD=D0=BE=D0=B2?= Date: Tue, 29 Sep 2026 23:03:07 +0300 Subject: [PATCH 15/27] Add pristan to base task dependencies --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 8db0d30..fd2ca92 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ dependencies = [ 'pytest>=8.3.5,<10', 'throng>=0.0.3', 'skelet>=0.0.21', + 'pristan>=0.0.24', ] classifiers = [ "Operating System :: OS Independent", From c8dad3617d89fa68ed41fbb5b9e1ee5e43f75209 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=95=D0=B2=D0=B3=D0=B5=D0=BD=D0=B8=D0=B9=20=D0=91=D0=BB?= =?UTF-8?q?=D0=B8=D0=BD=D0=BE=D0=B2?= Date: Tue, 29 Sep 2026 23:03:19 +0300 Subject: [PATCH 16/27] Add pytest-cov dependency to dev requirements --- requirements_dev.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements_dev.txt b/requirements_dev.txt index 29bb427..904bc21 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -4,6 +4,8 @@ pytest-xdist==3.8.0; python_version >= '3.9' coverage==7.6.1; python_version == '3.8' coverage==7.6.10; python_version >= '3.9' coverage-pyver-pragma==0.4.0 +pytest-cov==5.0.0; python_version < '3.9' +pytest-cov==6.3.0; python_version >= '3.9' build==1.2.2.post1 mypy==1.14.1 pytest-mypy-testing==0.1.3 From 16265eee8e97009a67dd81ded4e7c2a1a6e11da0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=95=D0=B2=D0=B3=D0=B5=D0=BD=D0=B8=D0=B9=20=D0=91=D0=BB?= =?UTF-8?q?=D0=B8=D0=BD=D0=BE=D0=B2?= Date: Tue, 29 Sep 2026 23:04:11 +0300 Subject: [PATCH 17/27] Add coverage agent support for isolated workers --- throngtest/coverage.py | 157 +++++++++++++++++++++++++++++++ throngtest/coverage_transport.py | 90 ++++++++++++++++++ throngtest/protocol.py | 3 + throngtest/runner.py | 102 ++++++++++++++++---- throngtest/worker.py | 20 +++- 5 files changed, 354 insertions(+), 18 deletions(-) create mode 100644 throngtest/coverage.py create mode 100644 throngtest/coverage_transport.py diff --git a/throngtest/coverage.py b/throngtest/coverage.py new file mode 100644 index 0000000..210d064 --- /dev/null +++ b/throngtest/coverage.py @@ -0,0 +1,157 @@ +"""Coverage agents discovered through a pristan slot.""" + +import os +from abc import ABC, abstractmethod +from importlib import import_module +from pathlib import Path, PurePosixPath, PureWindowsPath +from typing import TYPE_CHECKING, Callable, Dict, List, Optional, cast + +import pytest +from pristan import slot + +from throngtest.coverage_transport import worker_data_file + +if TYPE_CHECKING: # pragma: no cover - this import only exists for static analysis. + from coverage import Coverage + + +class CoverageAgent(ABC): + """Adapt one coverage provider to the isolate lifecycle.""" + + @abstractmethod + def active(self, config: pytest.Config) -> bool: + ... # pragma: no cover - abstract signature. + + @abstractmethod + def start_worker(self, marker: str, configuration: Dict[str, object], root: Path) -> None: + ... # pragma: no cover - abstract signature. + + def configuration(self, _root: Path) -> Dict[str, object]: + """Return JSON-safe settings needed by this agent in an isolate.""" + return {} + + def pytest_arguments(self) -> List[str]: + """Override provider options that must be evaluated by the controller.""" + return [] + + @abstractmethod + def stop_worker(self) -> None: + ... # pragma: no cover - abstract signature. + + def data_file(self) -> str: + """Return the provider's configured coverage data file in this process.""" + from coverage import Coverage # noqa: PLC0415 + current = Coverage.current() + return str(current.config.data_file if current is not None else Coverage().config.data_file) + + +@slot(entrypoint_group='throngtest.coverage') +def coverage_agents() -> Dict[str, CoverageAgent]: # type: ignore[empty-body] + ... # pragma: no cover - pristan replaces the slot body. + + +class PythonCoverageAgent(CoverageAgent): + """Start coverage.py in a worker created by ``coverage run``.""" + + def __init__(self) -> None: + self.started: Optional['Coverage'] = None + self.config_path: Optional[Path] = None + + def active(self, config: pytest.Config) -> bool: + try: + from coverage import Coverage # noqa: PLC0415 + except ImportError: + return False + return Coverage.current() is not None and not bool(cast(Optional[List[str]], config.getoption('cov_source', default=None))) + + def configuration(self, root: Path) -> Dict[str, object]: + from coverage import Coverage # noqa: PLC0415 + current = Coverage.current() + assert current is not None + filename = current.config.config_file + path = Path(filename) if filename else None + return { + 'suffix': path.suffix if path else '', + 'text': path.read_text() if path and path.is_file() else '', + 'source': current.config.source, + 'branch': current.config.branch, + 'root': str(root), + } + + def start_worker(self, marker: str, configuration: Dict[str, object], root: Path) -> None: + from coverage import Coverage # noqa: PLC0415 + if Coverage.current() is None: + os.environ['COVERAGE_FILE'] = worker_data_file(marker) + contents = cast(str, configuration.get('text', '')) + if contents: + suffix = cast(str, configuration.get('suffix', '')) + self.config_path = Path('.throngtest-coverage-config-' + marker.rstrip(':') + suffix) + self.config_path.write_text(contents) + original_source = cast(Optional[List[str]], configuration.get('source')) + source = original_source + controller_root = cast(str, configuration.get('root', '')) + if controller_root and original_source: + controller_path = PureWindowsPath(controller_root) if '\\' in controller_root or ':' in controller_root else PurePosixPath(controller_root) + + def map_source(value: str) -> str: + try: + relative = type(controller_path)(value).relative_to(controller_path) + except ValueError: + return value + return str(root.joinpath(*relative.parts)) + + source = [map_source(value) for value in original_source] + self.started = Coverage( + config_file=str(self.config_path) if self.config_path else True, + source=source, + branch=cast(bool, configuration.get('branch')), + ) + self.started.start() + + def stop_worker(self) -> None: + if self.started is not None: + self.started.stop() + self.started.save() + if self.config_path is not None: + self.config_path.unlink() + + +class PytestCovAgent(CoverageAgent): + """Use pytest-cov's own worker lifecycle and collect its saved data.""" + + def __init__(self) -> None: + self.worker_file: Optional[str] = None + + def active(self, config: pytest.Config) -> bool: + return bool(cast(Optional[List[str]], config.getoption('cov_source', default=None))) and not cast(bool, config.getoption('no_cov', default=False)) + + def start_worker(self, marker: str, _configuration: Dict[str, object], _root: Path) -> None: + # pytest-cov 6 and earlier starts a subprocess tracer before this + # module runs. Stop it before selecting a per-isolate data file. + try: + cleanup = cast(Callable[[], None], import_module('pytest_cov.embed').cleanup) + except (ImportError, AttributeError): + pass + else: + cleanup() + self.worker_file = worker_data_file(marker) + os.environ['COVERAGE_FILE'] = self.worker_file + + def stop_worker(self) -> None: + pass + + def data_file(self) -> str: + return self.worker_file or super().data_file() + + def pytest_arguments(self) -> List[str]: + return ['--cov-fail-under=0'] + + +@coverage_agents.plugin(unique=True) +def python_coverage() -> CoverageAgent: + return PythonCoverageAgent() + + +@coverage_agents.plugin(unique=True) +def pytest_cov() -> CoverageAgent: + return PytestCovAgent() diff --git a/throngtest/coverage_transport.py b/throngtest/coverage_transport.py new file mode 100644 index 0000000..57e7ca2 --- /dev/null +++ b/throngtest/coverage_transport.py @@ -0,0 +1,90 @@ +"""Move coverage data through isolate command output, without shared storage.""" + +import base64 +import hashlib +import json +import sys +from pathlib import Path, PurePosixPath, PureWindowsPath +from tempfile import TemporaryDirectory +from typing import Dict, List, Set, cast +from uuid import uuid4 + +from throngtest.protocol import Request, WorkerError, encode + + +def marker_path(marker: str) -> Path: + return Path('.throngtest-coverage-' + hashlib.sha256(marker.encode()).hexdigest()) + + +def worker_data_file(marker: str) -> str: + return '.coverage.throngtest-' + hashlib.sha256(marker.encode()).hexdigest() + + +def export(value: str) -> None: + """Print all new coverage databases after the pytest process has exited.""" + request = Request.unpack(value) + root = Path.cwd().resolve() + directory = root / request.directory + marker = directory / marker_path(request.marker) + since = marker.stat().st_mtime_ns + data_files = cast(Dict[str, str], json.loads(marker.read_text())) + files: Dict[str, List[str]] = {} + for name in request.coverage_agents or []: + candidates: Set[Path] = set() + data_file = Path(data_files[name]) + if not data_file.is_absolute(): + data_file = directory / data_file + candidates.update(root.rglob(data_file.name + '*')) + candidates.update(data_file.parent.glob(data_file.name + '*')) + files[name] = [] + for path in sorted(candidates): + if path.is_file() and path.stat().st_mtime_ns >= since: + content = path.read_bytes() + if content.startswith(b'SQLite format 3\x00'): + files[name].append(base64.b64encode(content).decode('ascii')) + marker.unlink() + sys.stdout.write(request.marker + encode({'version': 1, 'root': str(root), 'files': files}) + '\n') + + +def receive(payload: Dict[str, object], targets: Dict[str, str], root: Path) -> None: + """Restore and remap coverage databases before the controller reports.""" + remote = payload.get('root') + files = payload.get('files') + if not isinstance(remote, str) or not isinstance(files, dict): + raise WorkerError('isolate did not return coverage data') + from coverage import CoverageData # noqa: PLC0415 + from coverage.exceptions import DataError # noqa: PLC0415 + remote_path = PureWindowsPath(remote) if '\\' in remote or ':' in remote else PurePosixPath(remote) + + def map_path(filename: str) -> str: + path = type(remote_path)(filename) + try: + relative = path.relative_to(remote_path) + except ValueError: + return filename + return str(root.joinpath(*relative.parts)) + + for name, target in targets.items(): + records = files.get(name) + if not isinstance(records, list) or not records: + raise WorkerError(f'isolate did not return coverage data for {name}') + for item in cast(List[object], records): + if not isinstance(item, str): + raise WorkerError('isolate returned malformed coverage data') + try: + content = base64.b64decode(item, validate=True) + except ValueError as error: + raise WorkerError('isolate returned malformed coverage data') from error + if not content.startswith(b'SQLite format 3\x00'): + raise WorkerError('isolate returned malformed coverage data') + try: + with TemporaryDirectory(prefix='throngtest-coverage-') as temporary: + source = Path(temporary) / 'data' + source.write_bytes(content) + source_data = CoverageData(basename=str(source)) + source_data.read() + restored = CoverageData(basename=target, suffix=uuid4().hex) + restored.update(source_data, map_path=map_path) + restored.write() + except DataError as error: + raise WorkerError('isolate returned malformed coverage data') from error diff --git a/throngtest/protocol.py b/throngtest/protocol.py index 05b5e74..3101809 100644 --- a/throngtest/protocol.py +++ b/throngtest/protocol.py @@ -34,6 +34,9 @@ class Request: shard: int marker: str directory: str + coverage_agents: Optional[List[str]] = None + coverage_targets: Optional[Dict[str, str]] = None + coverage_configurations: Optional[Dict[str, Dict[str, object]]] = None def pack(self) -> str: return encode(cast(Dict[str, object], vars(self))) diff --git a/throngtest/runner.py b/throngtest/runner.py index 2742069..3859669 100644 --- a/throngtest/runner.py +++ b/throngtest/runner.py @@ -5,14 +5,17 @@ import warnings from collections import Counter from concurrent.futures import ThreadPoolExecutor, as_completed +from contextlib import contextmanager from pathlib import Path -from typing import Dict, List, Optional, Sequence, cast +from typing import Dict, Iterator, List, Optional, Sequence, Tuple, cast from uuid import uuid4 import pytest from cantok import SimpleToken from throng import AbstractManager, throng +from throngtest.coverage import coverage_agents +from throngtest.coverage_transport import receive from throngtest.distribution import collection_difference, fingerprint, partition from throngtest.protocol import Request, WorkerError, read_response from throngtest.settings import ARGUMENTS, Settings @@ -39,12 +42,78 @@ def relocate(argument: str, root: Path, invocation: Path, directory: Path) -> st return prefix + separator + value +def coverage_plan(config: pytest.Config, backend: str) -> Tuple[List[str], Optional[Dict[str, str]], Optional[Dict[str, Dict[str, object]]]]: + """Select every active coverage agent and its controller-side data file.""" + agents = coverage_agents() + enabled = [name for name, agent in agents.items() if agent.active(config)] + if shared_local_coverage(backend): + # coverage.py already starts in each local child and writes to the + # explicit controller path. Other active agents still participate. + enabled = [name for name in enabled if name != 'python_coverage'] + if not enabled: + return [], None, None + return enabled, {name: str(Path(agents[name].data_file()).absolute()) for name in enabled}, {name: agents[name].configuration(config.rootpath) for name in enabled} + + +def shared_local_coverage(backend: str) -> bool: + """Detect coverage.py startup with an explicit same-machine data path.""" + data_file = os.environ.get('COVERAGE_FILE') + return backend in ('local', 'temporary_directory') and bool(os.environ.get('COVERAGE_PROCESS_START')) and bool(data_file and Path(data_file).is_absolute()) + + +@contextmanager +def isolate_coverage_environment(enabled: bool) -> Iterator[None]: + """Keep controller-only coverage paths out of potentially remote Python.""" + names = ('COVERAGE_PROCESS_START', 'COV_CORE_SOURCE', 'COV_CORE_CONFIG', 'COV_CORE_DATAFILE', 'COV_CORE_BRANCH', 'COV_CORE_CONTEXT') + saved = {name: os.environ.pop(name) for name in names if enabled and name in os.environ} + try: + yield + finally: + os.environ.update(saved) + + +def worker_arguments(config: pytest.Config, root: Path, invocation: Path, directory: Path) -> List[str]: + """Preserve effective pytest options while relocating project paths.""" + arguments: List[str] = [] + preserve_value = False + for argument in config.stash[ARGUMENTS]: + if preserve_value: + arguments.append(argument) + preserve_value = False + else: + arguments.append(relocate(argument, root, invocation, root / directory)) + preserve_value = argument == '--isolates' or ( + argument.startswith('--throngtest-') and '=' not in argument and argument not in ( + '--throngtest-check-fingerprints', '--throngtest-no-check-fingerprints', + ) + ) + arguments.extend(['--rootdir', os.path.relpath(root, root / directory)]) + return arguments + + +def backend_error(error: Exception, stage: str, request: Request, settings: Settings, preparation_output: str) -> WorkerError: + """Keep backend phase and exception chain visible in pytest's final exit.""" + detail = str(error) + if not isinstance(error, WorkerError): + detail = f'{type(error).__name__}: {error}' + cause = error.__cause__ + while cause is not None: + detail += f'\ncaused by {type(cause).__name__}: {cause}' + cause = cause.__cause__ + message = f'isolate {request.shard + 1} (backend={settings.backend}): {stage} failed: {detail}' + if preparation_output: + message += f'\npreparation output:\n{preparation_output}' + return WorkerError(message, exitcode=error.exitcode if isinstance(error, WorkerError) else 3) + + def execute(manager: AbstractManager, request: Request, settings: Settings, token: SimpleToken, nodeids: Sequence[str] = ()) -> Dict[str, object]: command = shlex.join([settings.python, '-c', 'from throngtest.worker import main; import sys; main(sys.argv[1])', request.pack()]) preparation_output = '' + stage = 'acquiring isolate' try: with manager.scope as isolate: for index, instruction in enumerate(settings.preparation, 1): + stage = f'running preparation command {index}' try: prepared = isolate.run(instruction, token=token) except Exception as error: @@ -55,11 +124,21 @@ def execute(manager: AbstractManager, request: Request, settings: Settings, toke f'isolate {request.shard + 1}: preparation command {index} failed ' f'with exit code {prepared.returncode}: {instruction}', ) + stage = 'running pytest worker' result = isolate.run(command, token=token) + if request.coverage_agents and result.returncode in (0, 1) and request.marker in [line[:len(request.marker)] for line in (result.stdout or '').splitlines()]: + stage = 'exporting coverage' + exported = isolate.run(shlex.join([settings.python, '-c', 'from throngtest.coverage_transport import export; import sys; export(sys.argv[1])', request.pack()]), token=token) + if exported.returncode != 0: + raise WorkerError(f'isolate could not export coverage: {exported.stderr or exported.stdout}') + coverage_payload = read_response(exported.stdout or '', request.marker) + stage = 'releasing isolate' except Exception as error: # Third-party plugins may raise their own exception types. - raise WorkerError(f'could not execute isolate: {error}\npreparation output:\n{preparation_output}') from error + raise backend_error(error, stage, request, settings, preparation_output) from error stdout = result.stdout or '' try: + if request.coverage_agents and result.returncode in (0, 1) and request.marker in [line[:len(request.marker)] for line in stdout.splitlines()]: + receive(coverage_payload, request.coverage_targets or {}, Path.cwd()) response = read_response(stdout, request.marker) # The decoded protocol is represented by the diagnostic below; avoid # flooding errors with its base64 payload. Preserve unrelated output. @@ -208,23 +287,11 @@ def pytest_runtestloop(self, session: pytest.Session) -> bool: directory = invocation.relative_to(root) except ValueError: directory = Path() - arguments = [] - preserve_value = False - for argument in session.config.stash[ARGUMENTS]: - if preserve_value: - arguments.append(argument) - preserve_value = False - else: - arguments.append(relocate(argument, root, invocation, root / directory)) - preserve_value = argument == '--isolates' or ( - argument.startswith('--throngtest-') and '=' not in argument and argument not in ( - '--throngtest-check-fingerprints', '--throngtest-no-check-fingerprints', - ) - ) - arguments.extend(['--rootdir', os.path.relpath(root, root / directory)]) + arguments = worker_arguments(session.config, root, invocation, directory) nodeids = [item.nodeid for item in session.items] shards = partition(nodeids, self.settings.workers, self.settings.distribution) collection_fingerprint = fingerprint(nodeids) if self.settings.check_fingerprints else None + enabled, targets, configurations = coverage_plan(session.config, self.settings.backend) completed = 0 token = SimpleToken() previous = Path.cwd() @@ -236,12 +303,13 @@ def pytest_runtestloop(self, session: pytest.Session) -> bool: if self.settings.backend not in managers: raise pytest.UsageError(f'throngtest: unknown backend {self.settings.backend!r}; available: {", ".join(sorted(managers))}') manager = managers[self.settings.backend] - with ThreadPoolExecutor(max_workers=len(shards), thread_name_prefix='throngtest') as pool: + with isolate_coverage_environment(bool(enabled)), ThreadPoolExecutor(max_workers=len(shards), thread_name_prefix='throngtest') as pool: try: futures = { pool.submit(execute, manager, Request( arguments, collection_fingerprint, len(shards), self.settings.distribution, index, f'THRONGTEST_{uuid4().hex}:', str(directory), + enabled, targets, configurations, ), self.settings, token, nodeids): shard for index, shard in enumerate(shards) } diff --git a/throngtest/worker.py b/throngtest/worker.py index a6c6616..b20924b 100644 --- a/throngtest/worker.py +++ b/throngtest/worker.py @@ -1,5 +1,6 @@ """Runs inside an isolate; never creates further isolates.""" +import json import os import sys import warnings @@ -12,6 +13,8 @@ from _pytest._io import TerminalWriter from _pytest.terminal import TerminalReporter +from throngtest.coverage import coverage_agents +from throngtest.coverage_transport import marker_path from throngtest.distribution import fingerprint, partition from throngtest.protocol import Request, decode, encode from throngtest.settings import WORKER @@ -114,12 +117,27 @@ def pytest_warning_recorded(self, warning_message: warnings.WarningMessage, when def main(value: str) -> None: request = Request.unpack(value) + root = Path.cwd().resolve() os.chdir(request.directory) + agents = coverage_agents() + selected_names = request.coverage_agents or [] + if selected_names: + marker_path(request.marker).touch() + for name in selected_names: + agents[name].start_worker(request.marker, (request.coverage_configurations or {}).get(name, {}), root) + if selected_names: + data_files: Dict[str, str] = {name: agents[name].data_file() for name in selected_names} + marker_path(request.marker).write_text(json.dumps(data_files)) # Effective arguments already include ini addopts and PYTEST_ADDOPTS. os.environ.pop('PYTEST_ADDOPTS', None) with TemporaryDirectory(prefix='throngtest-pytest-') as basetemp: worker = Worker(request, basetemp) - status = pytest.main([*request.arguments, '-o', 'addopts='], plugins=[worker]) + try: + agent_arguments = [argument for name in selected_names for argument in agents[name].pytest_arguments()] + status = pytest.main([*request.arguments, *agent_arguments, '-o', 'addopts='], plugins=[worker]) + finally: + for name in selected_names: + agents[name].stop_worker() if status == pytest.ExitCode.NO_TESTS_COLLECTED and worker.assigned == []: status = pytest.ExitCode.OK if worker.parallel and worker.error and not worker.reports: From 61796a3c0ba12b2e0a648d95612f8dfade51b6c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=95=D0=B2=D0=B3=D0=B5=D0=BD=D0=B8=D0=B9=20=D0=91=D0=BB?= =?UTF-8?q?=D0=B8=D0=BD=D0=BE=D0=B2?= Date: Tue, 29 Sep 2026 23:04:54 +0300 Subject: [PATCH 18/27] Add comprehensive tests for coverage agents and transport --- tests/test_coverage_agents.py | 556 ++++++++++++++++++++++++++++++++++ tests/test_integration.py | 3 +- tests/test_preparation.py | 2 +- tests/test_runner.py | 29 +- tests/test_xdist.py | 2 +- 5 files changed, 587 insertions(+), 5 deletions(-) create mode 100644 tests/test_coverage_agents.py diff --git a/tests/test_coverage_agents.py b/tests/test_coverage_agents.py new file mode 100644 index 0000000..3a5e76d --- /dev/null +++ b/tests/test_coverage_agents.py @@ -0,0 +1,556 @@ +"""Coverage agents and transport tests.""" + +import base64 +import builtins +import json +import os +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace +from typing import Dict, List, cast +from unittest.mock import MagicMock + +import coverage +import pytest +from cantok import SimpleToken +from coverage import CoverageData + +from throngtest.coverage import ( + CoverageAgent, + PytestCovAgent, + PythonCoverageAgent, + coverage_agents, +) +from throngtest.coverage_transport import export, marker_path, receive, worker_data_file +from throngtest.protocol import Request, WorkerError, encode, read_response +from throngtest.runner import ( + coverage_plan, + execute, + isolate_coverage_environment, + shared_local_coverage, +) +from throngtest.settings import Settings +from throngtest.worker import main as worker_main + + +def test_builtin_agents_are_registered() -> None: + """Expose both coverage.py and pytest-cov through the pristan slot.""" + agents = coverage_agents() + assert isinstance(agents['python_coverage'], PythonCoverageAgent) + assert isinstance(agents['pytest_cov'], PytestCovAgent) + + +def test_coverage_plan_without_active_agents(pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch) -> None: + """Leave the protocol unchanged when no coverage provider is running.""" + monkeypatch.setattr('throngtest.runner.coverage_agents', dict) + assert coverage_plan(pytester.parseconfigure(), 'local') == ([], None, None) + + +def test_python_coverage_detection_and_configuration(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """Copy the running coverage configuration and avoid duplicate pytest-cov agents.""" + config_file = tmp_path / 'coverage.toml' + config_file.write_text('[run]\nbranch = true\n') + current = SimpleNamespace(config=SimpleNamespace(config_file=str(config_file), source=['package'], branch=True, data_file='.coverage')) + monkeypatch.setattr(coverage.Coverage, 'current', lambda: current) + agent = PythonCoverageAgent() + config = MagicMock() + config.getoption.return_value = [] + assert agent.active(config) + assert agent.configuration(tmp_path) == {'suffix': '.toml', 'text': config_file.read_text(), 'source': ['package'], 'branch': True, 'root': str(tmp_path)} + config.getoption.return_value = ['package'] + assert not agent.active(config) + monkeypatch.setattr(coverage.Coverage, 'current', lambda: None) + assert not agent.active(config) + + original_import = builtins.__import__ + + def without_coverage(name: str, *_args: object, **_kwargs: object) -> object: + if name == 'coverage': + raise ImportError('coverage is unavailable') + return original_import(name) + + monkeypatch.setattr(builtins, '__import__', without_coverage) + assert not agent.active(config) + + +def test_python_coverage_worker_lifecycle(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """Start a separate tracer with controller settings, then save and remove its config. + + A fake Coverage class keeps this unit test independent of the tracer that + instruments the repository's own CI run. + """ + instances: List[object] = [] + + class FakeCoverage: + def __init__(self, **kwargs: object) -> None: + self.kwargs = kwargs + self.started = False + self.stopped = False + self.saved = False + self.config = SimpleNamespace(data_file='.coverage.default') + instances.append(self) + + @staticmethod + def current() -> None: + return None + + def start(self) -> None: + self.started = True + + def stop(self) -> None: + self.stopped = True + + def save(self) -> None: + self.saved = True + + monkeypatch.setattr(coverage, 'Coverage', FakeCoverage) + monkeypatch.setenv('COVERAGE_FILE', 'original') + monkeypatch.chdir(tmp_path) + agent = PythonCoverageAgent() + agent.start_worker('marker:', {'suffix': '.toml', 'text': '[tool.coverage.run]\nbranch = true\n', 'source': ['app'], 'branch': True}, tmp_path) + assert os.environ['COVERAGE_FILE'] == worker_data_file('marker:') + assert agent.config_path is not None + assert agent.config_path.read_text().startswith('[tool.coverage.run]') + assert cast(FakeCoverage, instances[0]).kwargs == {'config_file': str(agent.config_path), 'source': ['app'], 'branch': True} + assert cast(FakeCoverage, instances[0]).started + config_path = agent.config_path + agent.stop_worker() + assert cast(FakeCoverage, instances[0]).stopped + assert cast(FakeCoverage, instances[0]).saved + assert not config_path.exists() + + plain = PythonCoverageAgent() + plain.start_worker('other:', {}, tmp_path) + assert cast(FakeCoverage, instances[1]).kwargs == {'config_file': True, 'source': None, 'branch': None} + assert plain.config_path is None + plain.stop_worker() + monkeypatch.setattr(FakeCoverage, 'current', lambda: object()) + inactive = PythonCoverageAgent() + inactive.start_worker('unused:', {}, tmp_path) + inactive.stop_worker() + assert len(instances) == 2 + + monkeypatch.setattr(FakeCoverage, 'current', lambda: None) + posix = PythonCoverageAgent() + posix.start_worker('posix:', {'root': '/controller/project', 'source': ['/controller/project/src', 'installed_package']}, tmp_path) + assert cast(FakeCoverage, instances[2]).kwargs['source'] == [str(tmp_path / 'src'), 'installed_package'] + posix.stop_worker() + windows = PythonCoverageAgent() + windows.start_worker('windows:', {'root': r'C:\controller\project', 'source': [r'C:\controller\project\src']}, tmp_path) + assert cast(FakeCoverage, instances[3]).kwargs['source'] == [str(tmp_path / 'src')] + windows.stop_worker() + + +def test_pytest_cov_worker_lifecycle(monkeypatch: pytest.MonkeyPatch) -> None: + """Replace an embedded subprocess tracer before pytest-cov starts its own.""" + cleanups: List[bool] = [] + monkeypatch.setenv('COVERAGE_FILE', 'original') + monkeypatch.setattr('throngtest.coverage.import_module', lambda _name: SimpleNamespace(cleanup=lambda: cleanups.append(True))) + agent = PytestCovAgent() + config = MagicMock() + config.getoption.side_effect = lambda key, **_kwargs: ['app'] if key == 'cov_source' else False + assert agent.active(config) + assert agent.pytest_arguments() == ['--cov-fail-under=0'] + agent.start_worker('marker:', {}, Path.cwd()) + assert cleanups == [True] + assert agent.data_file() == worker_data_file('marker:') + assert os.environ['COVERAGE_FILE'] == agent.data_file() + agent.stop_worker() + config.getoption.side_effect = lambda key, **_kwargs: True if key == 'no_cov' else ['app'] + assert not agent.active(config) + + def missing_embed(_name: str) -> object: + raise ImportError('pytest-cov 7 has no embed module') + + monkeypatch.setattr('throngtest.coverage.import_module', missing_embed) + agent.start_worker('second:', {}, Path.cwd()) + assert agent.data_file() == worker_data_file('second:') + + +def test_coverage_agent_defaults(monkeypatch: pytest.MonkeyPatch) -> None: + """Provide default arguments and data-file discovery for external agents.""" + class Extra(CoverageAgent): + def active(self, _config: pytest.Config) -> bool: + return False + + def start_worker(self, _marker: str, _configuration: Dict[str, object], _root: Path) -> None: + pass + + def stop_worker(self) -> None: + pass + + extra = Extra() + assert extra.configuration(Path.cwd()) == {} + assert extra.pytest_arguments() == [] + current = SimpleNamespace(config=SimpleNamespace(data_file='current.db')) + monkeypatch.setattr(coverage.Coverage, 'current', lambda: current) + assert extra.data_file() == 'current.db' + class NoCurrentCoverage: + def __init__(self) -> None: + self.config = SimpleNamespace(data_file='default.db') + + @staticmethod + def current() -> None: + return None + + monkeypatch.setattr(coverage, 'Coverage', NoCurrentCoverage) + assert extra.data_file() == 'default.db' + + +def test_external_agent_is_loaded_from_pristan_entrypoint(tmp_path: Path) -> None: + """Discover a third-party agent through the documented entry-point group. + + Discovery runs in a child process so registration cannot leak into other + tests through pristan's process-global slot. + """ + (tmp_path / 'extra_agent.py').write_text(''' +from throngtest.coverage import CoverageAgent, coverage_agents +class Extra(CoverageAgent): + def active(self, config): return False + def start_worker(self, marker, configuration, root): pass + def stop_worker(self): pass +@coverage_agents.plugin(unique=True) +def extra(): return Extra() +''') + metadata = tmp_path / 'throngtest_extra-0.0.0.dist-info' + metadata.mkdir() + (metadata / 'METADATA').write_text('Name: throngtest-extra\nVersion: 0.0.0\n') + (metadata / 'entry_points.txt').write_text('[throngtest.coverage]\nextra = extra_agent:extra\n') + environment = dict(os.environ) + environment['PYTHONPATH'] = str(tmp_path) + os.pathsep + environment.get('PYTHONPATH', '') + result = subprocess.run([sys.executable, '-c', 'from throngtest.coverage import coverage_agents; print(sorted(coverage_agents()))'], cwd=tmp_path, env=environment, text=True, capture_output=True, check=False) + assert result.returncode == 0, result.stdout + result.stderr + assert "['extra', 'pytest_cov', 'python_coverage']" in result.stdout + + +def test_controller_coverage_paths_are_not_passed_to_isolates(monkeypatch: pytest.MonkeyPatch) -> None: + """Hide controller paths only during isolate commands and restore them afterward.""" + monkeypatch.setenv('COVERAGE_PROCESS_START', '/controller/pyproject.toml') + monkeypatch.setenv('COV_CORE_DATAFILE', '/controller/.coverage') + with isolate_coverage_environment(False): + assert os.environ['COVERAGE_PROCESS_START'] == '/controller/pyproject.toml' + with isolate_coverage_environment(True): + assert 'COVERAGE_PROCESS_START' not in os.environ + assert 'COV_CORE_DATAFILE' not in os.environ + assert os.environ['COVERAGE_PROCESS_START'] == '/controller/pyproject.toml' + assert os.environ['COV_CORE_DATAFILE'] == '/controller/.coverage' + + +def test_existing_shared_local_coverage_needs_no_transfer(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """Recognize the one coverage.py case already handled by local startup hooks.""" + monkeypatch.delenv('COVERAGE_PROCESS_START', raising=False) + monkeypatch.delenv('COVERAGE_FILE', raising=False) + assert not shared_local_coverage('local') + monkeypatch.setenv('COVERAGE_PROCESS_START', str(tmp_path / 'pyproject.toml')) + monkeypatch.setenv('COVERAGE_FILE', '.coverage') + assert not shared_local_coverage('temporary_directory') + monkeypatch.setenv('COVERAGE_FILE', str(tmp_path / '.coverage')) + assert shared_local_coverage('local') + assert shared_local_coverage('temporary_directory') + assert not shared_local_coverage('remote_backend') + + +def test_all_active_agents_are_selected(pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """Select every attached agent, including third-party implementations. + + A fake pristan result avoids changing the process-global slot during the + remaining test suite and verifies that both providers stay in the plan. + """ + class ExtraAgent(CoverageAgent): + def active(self, _config: pytest.Config) -> bool: + return True + + def start_worker(self, _marker: str, _configuration: Dict[str, object], _root: Path) -> None: + pass + + def stop_worker(self) -> None: + pass + + def data_file(self) -> str: + return str(tmp_path / 'extra.db') + + agents = {'first': ExtraAgent(), 'second': ExtraAgent()} + monkeypatch.setattr('throngtest.runner.coverage_agents', lambda: agents) + enabled, targets, configurations = coverage_plan(pytester.parseconfigure(), 'local') + assert enabled == ['first', 'second'] + assert targets == {name: str(tmp_path / 'extra.db') for name in enabled} + assert configurations == {'first': {}, 'second': {}} + + +def test_shared_coverage_keeps_other_agents(pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """Skip only coverage.py when its local startup tracer already writes shared data.""" + class ActiveAgent(CoverageAgent): + def active(self, _config: pytest.Config) -> bool: + return True + + def start_worker(self, _marker: str, _configuration: Dict[str, object], _root: Path) -> None: + pass + + def stop_worker(self) -> None: + pass + + def data_file(self) -> str: + return str(tmp_path / '.coverage') + + monkeypatch.setattr('throngtest.runner.coverage_agents', lambda: {'python_coverage': ActiveAgent(), 'external': ActiveAgent()}) + monkeypatch.setenv('COVERAGE_PROCESS_START', str(tmp_path / 'pyproject.toml')) + monkeypatch.setenv('COVERAGE_FILE', str(tmp_path / '.coverage')) + assert coverage_plan(pytester.parseconfigure(), 'local')[0] == ['external'] + assert coverage_plan(pytester.parseconfigure(), 'temporary_directory')[0] == ['external'] + assert coverage_plan(pytester.parseconfigure(), 'remote')[0] == ['python_coverage', 'external'] + + +def test_worker_starts_all_selected_agents(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + """Start each selected agent before pytest and stop each after it exits. + + Replacing pytest.main keeps the protocol and lifecycle visible without + launching a second pytest session inside the current test process. + """ + events: List[str] = [] + + class Extra(CoverageAgent): + def __init__(self, name: str) -> None: + self.name = name + + def active(self, _config: pytest.Config) -> bool: + return True + + def start_worker(self, marker: str, configuration: Dict[str, object], root: Path) -> None: + events.append(f'start {self.name} {marker} {configuration["value"]}') + assert root == tmp_path + + def stop_worker(self) -> None: + events.append(f'stop {self.name}') + + def data_file(self) -> str: + return f'.coverage.{self.name}' + + def pytest_arguments(self) -> List[str]: + return [f'--agent={self.name}'] + + request = Request([], None, 1, 'tests', 0, 'marker:', '.', ['first', 'second'], coverage_configurations={'first': {'value': 1}, 'second': {'value': 2}}) + monkeypatch.chdir(tmp_path) + monkeypatch.setenv('PYTEST_ADDOPTS', '--wrong-option') + monkeypatch.setattr('throngtest.worker.coverage_agents', lambda: {'first': Extra('first'), 'second': Extra('second')}) + + def fake_pytest_main(arguments: List[str], plugins: List[object]) -> pytest.ExitCode: + assert arguments == ['--agent=first', '--agent=second', '-o', 'addopts='] + assert 'PYTEST_ADDOPTS' not in os.environ + assert len(plugins) == 1 + assert json.loads(marker_path(request.marker).read_text()) == {'first': '.coverage.first', 'second': '.coverage.second'} + events.append('pytest') + return pytest.ExitCode.OK + + monkeypatch.setattr(pytest, 'main', fake_pytest_main) + with pytest.raises(SystemExit) as caught: + worker_main(request.pack()) + assert caught.value.code == 0 + assert events == ['start first marker: 1', 'start second marker: 2', 'pytest', 'stop first', 'stop second'] + assert read_response(capsys.readouterr().out, request.marker)['exitcode'] == 0 + + +@pytest.mark.parametrize('exitcode', [0, 1]) +def test_coverage_export_is_received_before_replay(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, exitcode: int) -> None: + """Fetch coverage after passing or failing tests, before replaying reports.""" + manager = MagicMock() + request = Request([], None, 1, 'tests', 0, 'marker:', '.', ['provider'], {'provider': str(tmp_path / '.coverage')}) + manager.scope.__enter__.return_value.run.side_effect = [ + SimpleNamespace(stdout='before\nmarker:' + encode({'version': 1, 'exitcode': exitcode}) + '\n', stderr='', returncode=exitcode), + SimpleNamespace(stdout='marker:' + encode({'version': 1, 'root': '/remote', 'files': {'provider': ['encoded']}}), stderr='', returncode=0), + ] + received: List[object] = [] + monkeypatch.setattr('throngtest.runner.receive', lambda payload, targets, root: received.append((payload, targets, root))) + response = execute(manager, request, Settings(_sources=[]), SimpleToken()) + assert response['exitcode'] == exitcode + assert response['output'] == 'before' + assert received == [({'version': 1, 'root': '/remote', 'files': {'provider': ['encoded']}}, request.coverage_targets, Path.cwd())] + assert manager.scope.__enter__.return_value.run.call_count == 2 + + +@pytest.mark.parametrize('remote_name', ['/remote/project/app.py', r'C:\remote\project\app.py']) +def test_receive_restores_data_without_remote_files(tmp_path: Path, remote_name: str) -> None: + """Rebuild coverage locally after the remote directory has disappeared. + + The remote database exists only long enough to become stdout payload; + path remapping is checked for POSIX and Windows worker paths. + """ + remote = '/remote/project' if remote_name.startswith('/') else r'C:\remote\project' + source = CoverageData(basename=str(tmp_path / 'remote-data')) + source.add_lines({remote_name: {2}}) + source.write() + payload: Dict[str, object] = { + 'root': remote, + 'files': {'provider': [base64.b64encode((tmp_path / 'remote-data').read_bytes()).decode('ascii')]}, + } + (tmp_path / 'remote-data').unlink() + target = tmp_path / '.coverage' + receive(payload, {'provider': str(target)}, tmp_path) + restored = list(tmp_path.glob('.coverage.*')) + assert len(restored) == 1 + data = CoverageData(basename=str(restored[0])) + data.read() + assert data.lines(str(tmp_path / 'app.py')) == [2] + + +def test_receive_preserves_sources_outside_remote_project(tmp_path: Path) -> None: + """Keep external source paths when remapping only the isolate project root.""" + source = CoverageData(basename=str(tmp_path / 'remote-data')) + source.add_lines({'/external/library.py': {3}}) + source.write() + payload: Dict[str, object] = { + 'root': '/remote/project', + 'files': {'provider': [base64.b64encode((tmp_path / 'remote-data').read_bytes()).decode('ascii')]}, + } + receive(payload, {'provider': str(tmp_path / '.coverage')}, tmp_path) + restored = CoverageData(basename=str(next(tmp_path.glob('.coverage.*')))) + restored.read() + assert restored.lines('/external/library.py') == [3] + + +def test_worker_data_file_is_unique_per_isolate() -> None: + """Give each isolate a stable, distinct data-file prefix.""" + assert worker_data_file('first:') == worker_data_file('first:') + assert worker_data_file('first:') != worker_data_file('second:') + + +def test_export_ignores_stale_data_and_removes_marker(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: + """Send only databases produced after the worker starts. + + Existing coverage files may be present in the copied project, so the + marker separates this run's data from stale input. + """ + request = Request([], None, 1, 'tests', 0, 'unique:', '.', ['provider']) + monkeypatch.chdir(tmp_path) + old = CoverageData(basename=str(tmp_path / '.coverage.old')) + old.add_lines({str(tmp_path / 'old.py'): {1}}) + old.write() + os.utime(tmp_path / '.coverage.old', (1, 1)) + marker_path(request.marker).write_text(json.dumps({'provider': '.coverage'})) + new = CoverageData(basename=str(tmp_path / '.coverage.new')) + new.add_lines({str(tmp_path / 'new.py'): {2}}) + new.write() + export(request.pack()) + response = read_response(capsys.readouterr().out, request.marker) + assert len(cast(Dict[str, list], response['files'])['provider']) == 1 + assert not marker_path(request.marker).exists() + + +def test_export_accepts_absolute_data_file_and_skips_non_sqlite(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: + """Find an absolute provider data path and ignore adjacent non-databases.""" + request = Request([], None, 1, 'tests', 0, 'absolute:', '.', ['provider']) + monkeypatch.chdir(tmp_path) + data_file = tmp_path / '.coverage.absolute' + marker_path(request.marker).write_text(json.dumps({'provider': str(data_file)})) + source = CoverageData(basename=str(data_file)) + source.add_lines({str(tmp_path / 'app.py'): {4}}) + source.write() + (tmp_path / '.coverage.absolute-junk').write_text('not a database') + export(request.pack()) + response = read_response(capsys.readouterr().out, request.marker) + assert len(cast(Dict[str, list], response['files'])['provider']) == 1 + + +@pytest.mark.parametrize('payload', [ + {}, + {'root': '/remote', 'files': []}, + {'root': '/remote', 'files': {}}, + {'root': '/remote', 'files': {'provider': []}}, + {'root': '/remote', 'files': {'provider': [42]}}, + {'root': '/remote', 'files': {'provider': ['bad-base64!']}}, + {'root': '/remote', 'files': {'provider': [base64.b64encode(b'not sqlite').decode('ascii')]}}, + {'root': '/remote', 'files': {'provider': [base64.b64encode(b'SQLite format 3\x00garbage').decode('ascii')]}}, +]) +def test_bad_coverage_payload_is_rejected(tmp_path: Path, payload: Dict[str, object]) -> None: + """Never mistake missing or malformed remote data for successful coverage.""" + with pytest.raises(WorkerError, match='coverage data'): + receive(payload, {'provider': str(tmp_path / '.coverage')}, tmp_path) + + +def test_export_failure_is_reported() -> None: + """Fail the run when the isolate cannot return coverage after its tests.""" + manager = MagicMock() + manager.scope.__enter__.return_value.run.side_effect = [ + SimpleNamespace(stdout='marker:', stderr='', returncode=0), + SimpleNamespace(stdout='', stderr='transfer failed', returncode=7), + ] + request = Request([], None, 1, 'tests', 0, 'marker:', '.', ['provider'], {'provider': '.coverage'}) + with pytest.raises(WorkerError, match='transfer failed'): + execute(manager, request, Settings(_sources=[]), SimpleToken()) + + +@pytest.mark.parametrize('backend', ['local', 'temporary_directory']) +@pytest.mark.parametrize('provider', ['coverage', 'pytest-cov']) +def test_real_coverage_reaches_controller(pytester: pytest.Pytester, backend: str, provider: str) -> None: + """Combine both isolate subsets after temporary copies are destroyed. + + Child processes do not inherit the repository's coverage startup or data + path. Each branch is exercised only in an isolate, so 100% proves that its + database crossed the command-output transport and was remapped correctly. + """ + pytester.makepyfile(app=''' + def classify(value): + if value > 0: + return 'positive' + return 'other' + ''', test_positive=''' + from app import classify + def test_positive(): assert classify(1) == 'positive' + ''', test_other=''' + from app import classify + def test_other(): assert classify(0) == 'other' + ''') + pytester.makefile('.toml', pyproject=''' + [tool.coverage.run] + source = ["app"] + branch = true + parallel = true + ''') + environment = dict(os.environ) + environment.pop('COVERAGE_PROCESS_START', None) + environment.pop('COVERAGE_FILE', None) + arguments = [sys.executable, '-m', 'coverage', 'run', '-m', 'pytest'] if provider == 'coverage' else [sys.executable, '-m', 'pytest', '--cov=app', '--cov-branch', '--cov-fail-under=100'] + result = subprocess.run([*arguments, '-q', '--isolates=2', f'--throngtest-backend={backend}'], cwd=pytester.path, env=environment, text=True, capture_output=True, check=False) + assert result.returncode == 0, result.stdout + result.stderr + if provider == 'coverage': + combined = subprocess.run([sys.executable, '-m', 'coverage', 'combine', '-q'], cwd=pytester.path, env=environment, text=True, capture_output=True, check=False) + assert combined.returncode == 0, combined.stdout + combined.stderr + report = subprocess.run([sys.executable, '-m', 'coverage', 'json', '-o', 'report.json'], cwd=pytester.path, env=environment, text=True, capture_output=True, check=False) + assert report.returncode == 0, report.stdout + report.stderr + data = json.loads((pytester.path / 'report.json').read_text()) + assert data['totals']['percent_covered'] == 100 + assert data['files']['app.py']['missing_branches'] == [] + + +def test_pytest_cov_combines_nested_xdist_isolates(pytester: pytest.Pytester) -> None: + """Aggregate coverage when xdist also distributes tests inside each isolate. + + Each of the two throng isolates starts two xdist workers. The controller + must see all four contributions before enforcing the 100% threshold. + """ + pytest.importorskip('xdist') + pytester.makepyfile(app=''' + def classify(value): + if value > 0: + return 'positive' + return 'other' + ''', test_positive=''' + from app import classify + def test_positive(): assert classify(1) == 'positive' + ''', test_other=''' + from app import classify + def test_other(): assert classify(0) == 'other' + ''') + environment = dict(os.environ) + environment.pop('COVERAGE_PROCESS_START', None) + environment.pop('COVERAGE_FILE', None) + result = subprocess.run([ + sys.executable, '-m', 'pytest', '-q', '--isolates=2', '--throngtest-backend=temporary_directory', + '-n', '2', '--cov=app', '--cov-branch', '--cov-fail-under=100', + ], cwd=pytester.path, env=environment, text=True, capture_output=True, check=False) + assert result.returncode == 0, result.stdout + result.stderr + report = subprocess.run([sys.executable, '-m', 'coverage', 'json', '-o', 'report.json'], cwd=pytester.path, env=environment, text=True, capture_output=True, check=False) + assert report.returncode == 0, report.stdout + report.stderr + data = json.loads((pytester.path / 'report.json').read_text()) + assert data['totals']['percent_covered'] == 100 + assert data['files']['app.py']['missing_branches'] == [] diff --git a/tests/test_integration.py b/tests/test_integration.py index 5f933f4..934c7de 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -27,7 +27,8 @@ def test_real_isolates_execute_every_test_once(pytester: pytest.Pytester, backen def observe(cls): original = cls.run def traced(self, *args, **kwargs): - (Path({str(tmp_path)!r}) / ('dispatch-' + uuid4().hex)).write_text(cls.__name__) + if 'from throngtest.worker import main' in args[0]: + (Path({str(tmp_path)!r}) / ('dispatch-' + uuid4().hex)).write_text(cls.__name__) return original(self, *args, **kwargs) cls.run = traced observe(LocalIsolate) diff --git a/tests/test_preparation.py b/tests/test_preparation.py index d1e567d..ccf1aea 100644 --- a/tests/test_preparation.py +++ b/tests/test_preparation.py @@ -83,7 +83,7 @@ def test_prepared(index): assert len(traces) == 2 expected_class = 'LocalIsolate' if backend == 'local' else 'TemporaryDirectoryIsolate' for trace in traces: - calls = [json.loads(line) for line in trace] + calls = [json.loads(line) for line in trace if 'throngtest.coverage_transport import export' not in line] assert len(calls) == 3 assert [call[0] for call in calls] == [expected_class] * 3 assert [call[1] for call in calls[:2]] == commands diff --git a/tests/test_runner.py b/tests/test_runner.py index 7f0581c..a671375 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -39,7 +39,7 @@ def test_relocate_paths(tmp_path: Path) -> None: @pytest.mark.parametrize('stage', ['enter', 'run', 'exit']) def test_isolate_errors_are_reported_and_cleanup_attempted(stage: str) -> None: - """Wrap backend failures and attempt cleanup after successfully entering a scope. + """Name the failing backend phase and attempt cleanup after scope entry. The mocked context manager fails separately on entry, execution, and exit to check the cleanup obligations at each point in the isolate lifecycle. @@ -51,12 +51,37 @@ def test_isolate_errors_are_reported_and_cleanup_attempted(stage: str) -> None: isolate = scope.__enter__.return_value target = {'enter': scope.__enter__, 'run': isolate.run, 'exit': scope.__exit__}[stage] target.side_effect = RuntimeError('backend problem') - with pytest.raises(WorkerError, match='could not execute isolate: backend problem'): + with pytest.raises(WorkerError) as caught: execute(manager, Request([], '', 1, 'tests', 0, 'marker:', '.'), Settings(_sources=[]), SimpleToken()) + phase = {'enter': 'acquiring isolate', 'run': 'running pytest worker', 'exit': 'releasing isolate'}[stage] + assert f'{phase} failed: RuntimeError: backend problem' in str(caught.value) if stage != 'enter': scope.__exit__.assert_called_once() +def test_backend_error_preserves_underlying_connection_failure() -> None: + """Expose the socket error beneath a third-party isolate allocation failure. + + Backend exceptions can wrap a lower-level transport exception. The runner + must show both causes and the allocation phase, even though pytest reports + only the final WorkerError message. + """ + manager = Mock() + manager.scope.__enter__ = Mock() + manager.scope.__exit__ = Mock(return_value=False) + failure = RuntimeError('The connection was lost; commands are never replayed.') + failure.__cause__ = OSError('socket is already closed') + manager.scope.__enter__.side_effect = failure + settings = Settings(_sources=[]) + settings.backend = 'fission' + with pytest.raises(WorkerError) as caught: + execute(manager, Request([], None, 1, 'tests', 0, 'marker:', '.'), settings, SimpleToken()) + message = str(caught.value) + assert 'isolate 1 (backend=fission): acquiring isolate failed' in message + assert 'RuntimeError: The connection was lost; commands are never replayed.' in message + assert 'caused by OSError: socket is already closed' in message + + def test_exit_code_mismatch() -> None: """Reject a worker response whose exit code contradicts the process exit status.""" manager = MagicMock() diff --git a/tests/test_xdist.py b/tests/test_xdist.py index 4f323d3..6568a2a 100644 --- a/tests/test_xdist.py +++ b/tests/test_xdist.py @@ -58,7 +58,7 @@ def test_item(index, worker_id, request): assert len(records) == 16 assert len({record['pid'] for record in records}) == 4 assert {record['shard'] for record in records} == {0, 1} - assert len(list(tmp_path.glob('dispatch-*'))) == 4 # One preparation and one pytest command per isolate. + assert len([path for path in tmp_path.glob('dispatch-*') if 'throngtest.coverage_transport import export' not in path.read_text()]) == 4 # One preparation and one pytest command per isolate. for shard in (0, 1): assert {record['worker'] for record in records if record['shard'] == shard} == {'gw0', 'gw1'} if backend == 'temporary_directory': From ad8bb84680a77b3e2541b53d2a27832cd54b6c11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=95=D0=B2=D0=B3=D0=B5=D0=BD=D0=B8=D0=B9=20=D0=91=D0=BB?= =?UTF-8?q?=D0=B8=D0=BD=D0=BE=D0=B2?= Date: Tue, 29 Sep 2026 23:05:03 +0300 Subject: [PATCH 19/27] Add coverage support with coverage run and pytest-cov agents Also add per-isolate for remote backends and CI test --- README.md | 48 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 464a756..5f6a46c 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,46 @@ selection. The controller collects tests before creating isolates, so its environment must already support that initial collection. Preparation runs before collection inside each worker, not before controller collection. +## Coverage + +Throngtest includes two coverage agents, for `coverage run -m pytest` and +`pytest --cov`. They are pristan plugins in the `throngtest.coverage` slot. +Only agents whose coverage tool is active participate in a run. Coverage tools +are optional dependencies; install the one you intend to use in the controller +and in every isolate. + +```bash +coverage run -m pytest --isolates=4 +coverage combine +coverage report -m + +pytest --isolates=4 --cov=your_package +``` + +Each isolate saves its coverage database under a unique name. After its pytest +process exits, throngtest runs another command in the same isolate, serializes +the database into command output, and reconstructs it on the controller. Source +paths inside copied projects are mapped back to the controller's project path. +The transport requires no shared filesystem, including with throng backends +that run on remote machines. With `pytest-cov`, the final report includes the +transferred data automatically. With `coverage run`, combine the data before +reporting, as shown above. A `COVERAGE_FILE` path shared with isolates is not +required. + +Third-party agents can register a function returning a `CoverageAgent` subclass +with `@coverage_agents.plugin(unique=True)` and expose that registration module +through a `throngtest.coverage` entry point. Every active registered agent is +started in each isolate and has its coverage data returned to the controller. +An agent implements `active(config)`, `start_worker(marker, configuration, +root)`, and `stop_worker()`. It can override `configuration(root)` to send +JSON-safe settings, `pytest_arguments()` to adjust worker options, and +`data_file()` to name its coverage database. `root` is the project root on +the corresponding machine. + +Subprocesses created *inside* a test still need subprocess coverage support +from the chosen coverage tool. On remote machines, its Python environment also +needs the agent package installed. + ## Distribution and execution The controller collects and selects tests using pytest. It partitions the @@ -187,6 +227,9 @@ throng backends must provide stdout and a process return code. The selected interpreter must have throngtest, pytest, the project's dependencies, and required pytest plugins available by the end of preparation. Throngtest only installs packages when explicitly instructed through preparation commands. +For a remote backend, set `--throngtest-python` to an interpreter available in +the isolate, such as `python`; its default is the controller's absolute +`sys.executable` path, which is usually absent on a remote machine. The controller replays pytest reports, preserving assertion explanations, captured output, skip/xfail/xpass results, setup/teardown failures, durations, @@ -234,5 +277,6 @@ plugins, trace checks of their isolate APIs, complete/disjoint distribution, concurrency barriers, configuration precedence, preparation, native reports, cancellation, cleanup, and fault injection at the protocol/backend boundaries. The existing CI also checks statement and branch coverage across Python and OS versions. -Subprocess coverage uses the startup hook configured in the CI workflow; tests -keep coverage data outside temporary isolate directories so it survives cleanup. +The CI workflow uses a startup hook to measure its own local subprocesses. +Integration tests also verify that the coverage agents transfer data from +temporary isolates, including when pytest-xdist runs inside them. From 5e6fa6783634c85dd2af2c80024d5993c5d4cfc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=95=D0=B2=D0=B3=D0=B5=D0=BD=D0=B8=D0=B9=20=D0=91=D0=BB?= =?UTF-8?q?=D0=B8=D0=BD=D0=BE=D0=B2?= Date: Wed, 30 Sep 2026 19:47:48 +0300 Subject: [PATCH 20/27] logos --- docs/assets/23-bold-core.svg | 5 +++++ docs/assets/25-dual-mesh.svg | 5 +++++ docs/assets/26-segmented-hub.svg | 5 +++++ docs/assets/35-dense-infra.svg | 5 +++++ docs/assets/logo_1.svg | 5 +++++ 5 files changed, 25 insertions(+) create mode 100644 docs/assets/23-bold-core.svg create mode 100644 docs/assets/25-dual-mesh.svg create mode 100644 docs/assets/26-segmented-hub.svg create mode 100644 docs/assets/35-dense-infra.svg create mode 100644 docs/assets/logo_1.svg diff --git a/docs/assets/23-bold-core.svg b/docs/assets/23-bold-core.svg new file mode 100644 index 0000000..d8a959c --- /dev/null +++ b/docs/assets/23-bold-core.svg @@ -0,0 +1,5 @@ + +Variation 23: bold core +A bolder take with thicker shards and a denser, more prominent throng execution core. + + diff --git a/docs/assets/25-dual-mesh.svg b/docs/assets/25-dual-mesh.svg new file mode 100644 index 0000000..d1236fe --- /dev/null +++ b/docs/assets/25-dual-mesh.svg @@ -0,0 +1,5 @@ + +Variation 25: dual mesh +A richer three-ring network variation for a more infrastructural, parallel-execution feel. + + diff --git a/docs/assets/26-segmented-hub.svg b/docs/assets/26-segmented-hub.svg new file mode 100644 index 0000000..7f0b8a7 --- /dev/null +++ b/docs/assets/26-segmented-hub.svg @@ -0,0 +1,5 @@ + +Variation 26: segmented hub +A version that echoes the outer pytest shards again around the central execution hub. + + diff --git a/docs/assets/35-dense-infra.svg b/docs/assets/35-dense-infra.svg new file mode 100644 index 0000000..f9b20b8 --- /dev/null +++ b/docs/assets/35-dense-infra.svg @@ -0,0 +1,5 @@ + +Variation 35: dense infra +A richer infrastructure-style core with three network rings inside the same sharded pytest shell. + + diff --git a/docs/assets/logo_1.svg b/docs/assets/logo_1.svg new file mode 100644 index 0000000..b2d690a --- /dev/null +++ b/docs/assets/logo_1.svg @@ -0,0 +1,5 @@ + +Sharded ring +A four-color pytest sharding ring surrounds a throng network core. + + From 857644f9a8b478175119f593ff5a8ea82ad15632 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=95=D0=B2=D0=B3=D0=B5=D0=BD=D0=B8=D0=B9=20=D0=91=D0=BB?= =?UTF-8?q?=D0=B8=D0=BD=D0=BE=D0=B2?= Date: Wed, 30 Sep 2026 19:51:17 +0300 Subject: [PATCH 21/27] logos --- docs/assets/23-bold-core.svg | 5 ----- docs/assets/25-dual-mesh.svg | 5 ----- docs/assets/26-segmented-hub.svg | 5 ----- docs/assets/{35-dense-infra.svg => logo_2.svg} | 0 4 files changed, 15 deletions(-) delete mode 100644 docs/assets/23-bold-core.svg delete mode 100644 docs/assets/25-dual-mesh.svg delete mode 100644 docs/assets/26-segmented-hub.svg rename docs/assets/{35-dense-infra.svg => logo_2.svg} (100%) diff --git a/docs/assets/23-bold-core.svg b/docs/assets/23-bold-core.svg deleted file mode 100644 index d8a959c..0000000 --- a/docs/assets/23-bold-core.svg +++ /dev/null @@ -1,5 +0,0 @@ - -Variation 23: bold core -A bolder take with thicker shards and a denser, more prominent throng execution core. - - diff --git a/docs/assets/25-dual-mesh.svg b/docs/assets/25-dual-mesh.svg deleted file mode 100644 index d1236fe..0000000 --- a/docs/assets/25-dual-mesh.svg +++ /dev/null @@ -1,5 +0,0 @@ - -Variation 25: dual mesh -A richer three-ring network variation for a more infrastructural, parallel-execution feel. - - diff --git a/docs/assets/26-segmented-hub.svg b/docs/assets/26-segmented-hub.svg deleted file mode 100644 index 7f0b8a7..0000000 --- a/docs/assets/26-segmented-hub.svg +++ /dev/null @@ -1,5 +0,0 @@ - -Variation 26: segmented hub -A version that echoes the outer pytest shards again around the central execution hub. - - diff --git a/docs/assets/35-dense-infra.svg b/docs/assets/logo_2.svg similarity index 100% rename from docs/assets/35-dense-infra.svg rename to docs/assets/logo_2.svg From d9981645a2e01beb4fb6689b261149c49e6bf6de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=95=D0=B2=D0=B3=D0=B5=D0=BD=D0=B8=D0=B9=20=D0=91=D0=BB?= =?UTF-8?q?=D0=B8=D0=BD=D0=BE=D0=B2?= Date: Wed, 30 Sep 2026 19:58:50 +0300 Subject: [PATCH 22/27] Replace logo_2.svg with wide banner design --- docs/assets/logo_2.svg | 537 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 532 insertions(+), 5 deletions(-) diff --git a/docs/assets/logo_2.svg b/docs/assets/logo_2.svg index f9b20b8..a766400 100644 --- a/docs/assets/logo_2.svg +++ b/docs/assets/logo_2.svg @@ -1,5 +1,532 @@ - -Variation 35: dense infra -A richer infrastructure-style core with three network rings inside the same sharded pytest shell. - - + +Variation 35: dense infraA richer infrastructure-style core with three network rings inside the same sharded pytest shell.Variation 35: dense infraTHRONGTEST From 8c6fd12dd2459726e438e3b78646660402d15cf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=95=D0=B2=D0=B3=D0=B5=D0=BD=D0=B8=D0=B9=20=D0=91=D0=BB?= =?UTF-8?q?=D0=B8=D0=BD=D0=BE=D0=B2?= Date: Wed, 30 Sep 2026 19:59:09 +0300 Subject: [PATCH 23/27] Add project logo to README --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 5f6a46c..034f1d0 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +![logo](https://raw.githubusercontent.com/mutating/throngtest/develop/docs/assets/logo_2.svg) + # throngtest Run pytest test subsets in [throng](https://github.com/mutating/throng) isolates. From 996b40bf9b077e274e0f4c825af49036ae66077b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=95=D0=B2=D0=B3=D0=B5=D0=BD=D0=B8=D0=B9=20=D0=91=D0=BB?= =?UTF-8?q?=D0=B8=D0=BD=D0=BE=D0=B2?= Date: Thu, 1 Oct 2026 13:39:45 +0300 Subject: [PATCH 24/27] Remove redundant title from README --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 034f1d0..3e63586 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,5 @@ ![logo](https://raw.githubusercontent.com/mutating/throngtest/develop/docs/assets/logo_2.svg) -# throngtest - Run pytest test subsets in [throng](https://github.com/mutating/throng) isolates. Throngtest is an independent pytest plugin: it has its own options and does not depend on pytest-xdist or implement xdist's flags or fixtures. From 25c0bd11a5a34413308a58223181ac0e97c099bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=95=D0=B2=D0=B3=D0=B5=D0=BD=D0=B8=D0=B9=20=D0=91=D0=BB?= =?UTF-8?q?=D0=B8=D0=BD=D0=BE=D0=B2?= Date: Thu, 1 Oct 2026 13:41:34 +0300 Subject: [PATCH 25/27] Add project badges to README --- README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/README.md b/README.md index 3e63586..94a2fcd 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,23 @@ +
+ ⓘ + +[![Downloads](https://static.pepy.tech/badge/throngtest/month)](https://pepy.tech/project/throngtest) +[![Downloads](https://static.pepy.tech/badge/throngtest)](https://pepy.tech/project/throngtest) +[![Coverage Status](https://coveralls.io/repos/github/mutating/throngtest/badge.svg?branch=main)](https://coveralls.io/github/mutating/throngtest?branch=main) +[![Lines of code](https://sloc.xyz/github/mutating/throngtest/?category=code)](https://github.com/boyter/scc/) +[![Hits-of-Code](https://hitsofcode.com/github/mutating/throngtest?branch=main)](https://hitsofcode.com/github/mutating/throngtest/view?branch=main) +[![Test-Package](https://github.com/mutating/throngtest/actions/workflows/tests_and_coverage.yml/badge.svg)](https://github.com/mutating/throngtest/actions/workflows/tests_and_coverage.yml) +[![Python versions](https://img.shields.io/pypi/pyversions/throngtest.svg)](https://pypi.python.org/pypi/throngtest) +[![PyPI version](https://badge.fury.io/py/throngtest.svg)](https://badge.fury.io/py/throngtest) +[![Checked with mypy](http://www.mypy-lang.org/static/mypy_badge.svg)](http://mypy-lang.org/) +[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) +[![DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/mutating/throngtest) + +
+ ![logo](https://raw.githubusercontent.com/mutating/throngtest/develop/docs/assets/logo_2.svg) + Run pytest test subsets in [throng](https://github.com/mutating/throng) isolates. Throngtest is an independent pytest plugin: it has its own options and does not depend on pytest-xdist or implement xdist's flags or fixtures. From 59a7aabe2b81c22a878b35b2a08ffed844fa82ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=95=D0=B2=D0=B3=D0=B5=D0=BD=D0=B8=D0=B9=20=D0=91=D0=BB?= =?UTF-8?q?=D0=B8=D0=BD=D0=BE=D0=B2?= Date: Thu, 1 Oct 2026 13:54:44 +0300 Subject: [PATCH 26/27] Ensure temporary coverage data file is deleted --- throngtest/coverage_transport.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/throngtest/coverage_transport.py b/throngtest/coverage_transport.py index 57e7ca2..d0ba842 100644 --- a/throngtest/coverage_transport.py +++ b/throngtest/coverage_transport.py @@ -82,9 +82,12 @@ def map_path(filename: str) -> str: source = Path(temporary) / 'data' source.write_bytes(content) source_data = CoverageData(basename=str(source)) - source_data.read() - restored = CoverageData(basename=target, suffix=uuid4().hex) - restored.update(source_data, map_path=map_path) - restored.write() + try: + source_data.read() + restored = CoverageData(basename=target, suffix=uuid4().hex) + restored.update(source_data, map_path=map_path) + restored.write() + finally: + source_data.erase() except DataError as error: raise WorkerError('isolate returned malformed coverage data') from error From 62972bbfc0e7c357f26416ae3e32d85286107c70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=95=D0=B2=D0=B3=D0=B5=D0=BD=D0=B8=D0=B9=20=D0=91=D0=BB?= =?UTF-8?q?=D0=B8=D0=BD=D0=BE=D0=B2?= Date: Thu, 1 Oct 2026 13:54:51 +0300 Subject: [PATCH 27/27] Add test for releasing damaged coverage database before cleanup --- tests/test_coverage_agents.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/test_coverage_agents.py b/tests/test_coverage_agents.py index 3a5e76d..7bcad4d 100644 --- a/tests/test_coverage_agents.py +++ b/tests/test_coverage_agents.py @@ -6,9 +6,11 @@ import os import subprocess import sys +from contextlib import contextmanager from pathlib import Path +from tempfile import TemporaryDirectory from types import SimpleNamespace -from typing import Dict, List, cast +from typing import Dict, Iterator, List, cast from unittest.mock import MagicMock import coverage @@ -467,6 +469,27 @@ def test_bad_coverage_payload_is_rejected(tmp_path: Path, payload: Dict[str, obj receive(payload, {'provider': str(tmp_path / '.coverage')}, tmp_path) +def test_bad_coverage_database_is_released_before_cleanup(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Close a damaged SQLite database before removing its temporary directory. + + The wrapper checks the file before directory cleanup, mirroring the point + where Windows rejects deletion of a database with an open connection. + """ + @contextmanager + def checked_temporary_directory(prefix: str) -> Iterator[str]: + with TemporaryDirectory(prefix=prefix) as directory: + try: + yield directory + finally: + assert not (Path(directory) / 'data').exists() + + monkeypatch.setattr('throngtest.coverage_transport.TemporaryDirectory', checked_temporary_directory) + damaged = base64.b64encode(b'SQLite format 3\x00garbage').decode('ascii') + payload: Dict[str, object] = {'root': '/remote', 'files': {'provider': [damaged]}} + with pytest.raises(WorkerError, match='malformed coverage data'): + receive(payload, {'provider': str(tmp_path / '.coverage')}, tmp_path) + + def test_export_failure_is_reported() -> None: """Fail the run when the isolate cannot return coverage after its tests.""" manager = MagicMock()