diff --git a/.github/scripts/build_changelog.sh b/.github/scripts/build_changelog.sh index f27d87450..378131080 100644 --- a/.github/scripts/build_changelog.sh +++ b/.github/scripts/build_changelog.sh @@ -16,4 +16,6 @@ print(match.group(1)) PY )" -towncrier build --yes --version "${version}" +# towncrier comes from the dev extra in uv.lock, which uv lock has just +# refreshed with the new project version. +uv run --frozen --extra dev towncrier build --yes --version "${version}" diff --git a/.github/scripts/run_live_staging_tests.sh b/.github/scripts/run_live_staging_tests.sh index 39bc7a3cb..6a1217bd8 100755 --- a/.github/scripts/run_live_staging_tests.sh +++ b/.github/scripts/run_live_staging_tests.sh @@ -1,7 +1,9 @@ #!/usr/bin/env bash set -euo pipefail -python -m pytest \ +# Run from the environment uv.lock defines (make install), which includes +# pytest-xdist for the two workers. +uv run --frozen --extra dev python -m pytest \ -n 2 \ --dist load \ --maxschedchunk=1 \ diff --git a/.github/scripts/test_alembic_v2_lifecycle.sh b/.github/scripts/test_alembic_v2_lifecycle.sh index 1a3d58b87..eab5e3e07 100755 --- a/.github/scripts/test_alembic_v2_lifecycle.sh +++ b/.github/scripts/test_alembic_v2_lifecycle.sh @@ -2,6 +2,6 @@ set -euo pipefail -uv run pytest -q \ +uv run --frozen --extra dev pytest -q \ tests/unit/v2/test_alembic_v2.py \ tests/integration/test_alembic_v2_lifecycle.py diff --git a/.github/workflows/alembic-v1-check.yml b/.github/workflows/alembic-v1-check.yml index 43a2fef7c..3bb4dacf9 100644 --- a/.github/workflows/alembic-v1-check.yml +++ b/.github/workflows/alembic-v1-check.yml @@ -8,6 +8,10 @@ jobs: mysql-lifecycle: name: Alembic MySQL lifecycle runs-on: ubuntu-latest + env: + # uv installs uv.lock on the Cloud Run image's Python, not the 3.14 + # that .python-version selects. + UV_PYTHON: "3.12" services: mysql: image: mysql:8.4 @@ -30,17 +34,19 @@ jobs: uses: actions/setup-python@v5 with: python-version: "3.12" - - name: Install dependencies + - name: Setup uv + uses: astral-sh/setup-uv@v6 + - name: Install locked dependencies run: make install - name: Test Alembic configuration and MySQL lifecycle - run: python -m pytest tests/unit/data/test_alembic_baseline.py tests/unit/data/test_v1_database_migration.py tests/integration/test_alembic_mysql_lifecycle.py -q + run: uv run --frozen --extra dev python -m pytest tests/unit/data/test_alembic_baseline.py tests/unit/data/test_v1_database_migration.py tests/integration/test_alembic_mysql_lifecycle.py -q env: ALEMBIC_DATABASE_URL: mysql+pymysql://root:policyengine_test@127.0.0.1:3306/policyengine_alembic_test - name: Require database at all v1 heads - run: python -m alembic -c alembic-v1.ini current --check-heads + run: uv run --frozen --extra dev python -m alembic -c alembic-v1.ini current --check-heads env: ALEMBIC_DATABASE_URL: mysql+pymysql://root:policyengine_test@127.0.0.1:3306/policyengine_alembic_test - name: Require no ungenerated v1 operations - run: python -m alembic -c alembic-v1.ini check + run: uv run --frozen --extra dev python -m alembic -c alembic-v1.ini check env: ALEMBIC_DATABASE_URL: mysql+pymysql://root:policyengine_test@127.0.0.1:3306/policyengine_alembic_test diff --git a/.github/workflows/alembic-v2-check.yml b/.github/workflows/alembic-v2-check.yml index 4af38a66f..88163ad19 100644 --- a/.github/workflows/alembic-v2-check.yml +++ b/.github/workflows/alembic-v2-check.yml @@ -23,6 +23,9 @@ jobs: --health-timeout=5s --health-retries=20 env: + # uv installs uv.lock on the Cloud Run image's Python, not the 3.14 + # that .python-version selects. + UV_PYTHON: "3.12" V2_MIGRATION_DATABASE_URL: postgresql+psycopg://postgres:policyengine_v2_test@127.0.0.1:5432/policyengine_v2_alembic_test V2_ALEMBIC_DISPOSABLE_TEST: "1" steps: @@ -35,10 +38,10 @@ jobs: - name: Setup uv uses: astral-sh/setup-uv@v6 - name: Install locked dependencies - run: uv sync --frozen + run: uv sync --frozen --extra dev - name: Test generated-only v2 migration configuration and lifecycle run: bash .github/scripts/test_alembic_v2_lifecycle.sh - name: Require database at the v2 head - run: uv run alembic -c alembic-v2.ini current --check-heads + run: uv run --frozen --extra dev alembic -c alembic-v2.ini current --check-heads - name: Require no ungenerated v2 schema or data operations - run: uv run alembic -c alembic-v2.ini check + run: uv run --frozen --extra dev alembic -c alembic-v2.ini check diff --git a/.github/workflows/migrate-v1-cloud-sql.yml b/.github/workflows/migrate-v1-cloud-sql.yml index fbf22fe57..16dc5198e 100644 --- a/.github/workflows/migrate-v1-cloud-sql.yml +++ b/.github/workflows/migrate-v1-cloud-sql.yml @@ -56,8 +56,11 @@ jobs: service_account: ${{ vars.GCP_DB_MIGRATION_SERVICE_ACCOUNT }} - name: Set up GCloud uses: google-github-actions/setup-gcloud@v2 - - name: Install dependencies - run: make install + # make install syncs .venv from uv.lock, which the bare python this + # job's scripts run does not read. Moving this job to uv.lock is a + # separate, verified change; until then it installs unlocked. + - name: Install dependencies (unlocked) + run: pip install -e ".[dev]" - name: Start Cloud SQL Auth Proxy run: bash .github/scripts/start_cloud_sql_proxy.sh - name: Upgrade and verify v1 database diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 9b8659899..6341446ea 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -190,6 +190,10 @@ jobs: permissions: contents: read id-token: write + env: + # uv installs uv.lock on the Cloud Run image's Python, not the 3.14 + # that .python-version selects. + UV_PYTHON: "3.12" steps: - name: Checkout repo uses: actions/checkout@v4 @@ -197,6 +201,8 @@ jobs: uses: actions/setup-python@v5 with: python-version: "3.12" + - name: Set up uv + uses: astral-sh/setup-uv@v6 - name: Auth uses: google-github-actions/auth@v2 with: @@ -204,10 +210,10 @@ jobs: service_account: ${{ secrets.GCP_DEPLOY_SERVICE_ACCOUNT }} - name: Wait until PolicyEngine bundle version is available on PyPI run: .github/wait-for-pypi.sh - - name: Install dependencies + - name: Install locked dependencies run: make install - name: Run environment variable tests - run: pytest tests/env_variables/test_environment_variables.py + run: uv run --frozen --extra dev pytest tests/env_variables/test_environment_variables.py env: POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }} # Compatibility name required by PolicyEngine Core. The GitHub @@ -223,6 +229,10 @@ jobs: permissions: contents: read id-token: write + env: + # uv installs uv.lock on the Cloud Run image's Python, not the 3.14 + # that .python-version selects. + UV_PYTHON: "3.12" steps: - name: Checkout repo uses: actions/checkout@v4 @@ -230,6 +240,8 @@ jobs: uses: actions/setup-python@v5 with: python-version: "3.12" + - name: Set up uv + uses: astral-sh/setup-uv@v6 - name: Update package lists run: sudo apt-get update - name: Install system dependencies @@ -245,7 +257,7 @@ jobs: uses: google-github-actions/setup-gcloud@v2 with: project_id: policyengine-api - - name: Install dependencies + - name: Install locked dependencies run: make install - name: Test the API run: make test diff --git a/.github/workflows/process-v1-household-mirror-event.yml b/.github/workflows/process-v1-household-mirror-event.yml index 752ee63b1..2f45d1a4e 100644 --- a/.github/workflows/process-v1-household-mirror-event.yml +++ b/.github/workflows/process-v1-household-mirror-event.yml @@ -52,8 +52,11 @@ jobs: uses: actions/setup-python@v5 with: python-version: "3.12" - - name: Install dependencies - run: make install + # make install syncs .venv from uv.lock, which the bare python this + # job's scripts run does not read. Moving this job to uv.lock is a + # separate, verified change; until then it installs unlocked. + - name: Install dependencies (unlocked) + run: pip install -e ".[dev]" - name: Authenticate authorized database operator uses: google-github-actions/auth@v2 with: diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index af02524d3..255c11e3d 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -96,6 +96,10 @@ jobs: (github.repository == 'PolicyEngine/policyengine-api') && !(github.event.head_commit.message == 'Update PolicyEngine API') runs-on: ubuntu-latest + env: + # uv installs uv.lock on the Cloud Run image's Python, not the 3.14 + # that .python-version selects. + UV_PYTHON: "3.12" steps: - name: Generate GitHub App token id: app-token @@ -115,8 +119,6 @@ jobs: uses: astral-sh/setup-uv@v6 with: version: "0.13.4" - - name: Install changelog tool - run: pip install towncrier - name: Bump project version run: python .github/bump_version.py - name: Refresh uv lock @@ -229,6 +231,9 @@ jobs: CLOUD_RUN_MIN_INSTANCES: "0" CLOUD_RUN_SERVICE_MIN_INSTANCES: "0" CLOUD_RUN_MAX_INSTANCES: "1" + # uv installs uv.lock on the Cloud Run image's Python, not the 3.14 + # that .python-version selects. + UV_PYTHON: "3.12" permissions: contents: read id-token: write @@ -246,13 +251,15 @@ jobs: uses: actions/setup-python@v5 with: python-version: "3.12" + - name: Setup uv + uses: astral-sh/setup-uv@v6 - name: Update package lists run: sudo apt-get update - name: Install release test dependencies run: sudo apt-get install -y redis-server - name: Start release test Redis run: sudo systemctl start redis-server - - name: Install application dependencies + - name: Install locked application dependencies run: make install - name: Run release tests run: bash .github/scripts/run_release_tests.sh @@ -322,7 +329,7 @@ jobs: - name: Wait for Cloud Run staging health run: bash .github/scripts/health_check.sh "${{ steps.candidate.outputs.url }}/readiness-check" - name: Run staging integration tests - run: python -m pytest tests/integration/test_cloud_run_candidate.py tests/integration/test_live_v2_metadata.py tests/integration/test_live_v2_policies.py -v + run: uv run --frozen --extra dev python -m pytest tests/integration/test_cloud_run_candidate.py tests/integration/test_live_v2_metadata.py tests/integration/test_live_v2_policies.py -v env: API_BASE_URL: ${{ steps.candidate.outputs.url }} STAGING_API_TEST_PROBE_ID: cloud-run-${{ steps.cloud_run.outputs.revision_tag }} @@ -344,6 +351,10 @@ jobs: if: | (github.repository == 'PolicyEngine/policyengine-api') && (github.event.head_commit.message == 'Update PolicyEngine API') + env: + # uv installs uv.lock on the Cloud Run image's Python, not the 3.14 + # that .python-version selects. + UV_PYTHON: "3.12" steps: - name: Checkout repo uses: actions/checkout@v4 @@ -351,8 +362,10 @@ jobs: uses: actions/setup-python@v5 with: python-version: "3.12" - - name: Install live test dependencies - run: pip install pytest pytest-xdist httpx + - name: Setup uv + uses: astral-sh/setup-uv@v6 + - name: Install locked live test dependencies + run: make install - name: Wait for Cloud Run staging candidate health before live tests run: bash .github/scripts/health_check.sh "${{ needs.deploy-cloud-run-staging.outputs.url }}/readiness-check" - name: Run live staging tests in two pytest workers @@ -524,12 +537,21 @@ jobs: # daytime peak, so full traffic needs the extra headroom. CLOUD_RUN_SERVICE_MIN_INSTANCES: "2" CLOUD_RUN_MAX_INSTANCES: "8" + # uv installs uv.lock on the Cloud Run image's Python, not the 3.14 + # that .python-version selects. + UV_PYTHON: "3.12" permissions: contents: read id-token: write steps: - name: Checkout repo uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Setup uv + uses: astral-sh/setup-uv@v6 - name: GCP authentication uses: "google-github-actions/auth@v2" with: @@ -575,10 +597,10 @@ jobs: CLOUD_RUN_TAG: ${{ steps.cloud_run.outputs.revision_tag }} - name: Wait for Cloud Run candidate health run: bash .github/scripts/health_check.sh "${{ steps.candidate.outputs.url }}/readiness-check" - - name: Install Cloud Run smoke test dependencies - run: pip install pytest httpx + - name: Install locked Cloud Run smoke test dependencies + run: make install - name: Run Cloud Run candidate smoke tests - run: python -m pytest tests/integration/test_cloud_run_candidate.py tests/integration/test_live_v2_metadata.py -v + run: uv run --frozen --extra dev python -m pytest tests/integration/test_cloud_run_candidate.py tests/integration/test_live_v2_metadata.py -v env: API_BASE_URL: ${{ steps.candidate.outputs.url }} STAGING_API_TEST_PROBE_ID: cloud-run-${{ steps.cloud_run.outputs.revision_tag }} diff --git a/.github/workflows/v2-integration-check.yml b/.github/workflows/v2-integration-check.yml index 3f402ae80..f6ebc8262 100644 --- a/.github/workflows/v2-integration-check.yml +++ b/.github/workflows/v2-integration-check.yml @@ -48,6 +48,9 @@ jobs: --health-timeout=5s --health-retries=20 env: + # uv installs uv.lock on the Cloud Run image's Python, not the 3.14 + # that .python-version selects. + UV_PYTHON: "3.12" ALEMBIC_DATABASE_URL: mysql+pymysql://root:policyengine_test@127.0.0.1:3306/policyengine_alembic_test V2_MIGRATION_DATABASE_URL: postgresql+psycopg://postgres:policyengine_v2_test@127.0.0.1:5432/policyengine_v2_alembic_test V2_ALEMBIC_DISPOSABLE_TEST: "1" @@ -64,37 +67,37 @@ jobs: - name: Setup uv uses: astral-sh/setup-uv@v6 - name: Install locked dependencies - run: uv sync --frozen + run: uv sync --frozen --extra dev - name: Prepare the disposable v1 schema - run: uv run alembic -c alembic-v1.ini upgrade head + run: uv run --frozen --extra dev alembic -c alembic-v1.ini upgrade head - name: Prepare the disposable v2 schema - run: uv run alembic -c alembic-v2.ini upgrade head + run: uv run --frozen --extra dev alembic -c alembic-v2.ini upgrade head - name: Create the managed v2 roles and grant their declared privileges - run: uv run python scripts/reconcile_v2_privileges.py apply --create-missing-roles + run: uv run --frozen --extra dev python scripts/reconcile_v2_privileges.py apply --create-missing-roles - name: Verify the managed v2 roles hold their declared privileges - run: uv run python scripts/reconcile_v2_privileges.py verify + run: uv run --frozen --extra dev python scripts/reconcile_v2_privileges.py verify - name: Verify the installed PolicyEngine.py catalog interface - run: uv run coverage run --branch -m pytest -q tests/integration/test_v2_catalog_installed.py + run: uv run --frozen --extra dev coverage run --branch -m pytest -q tests/integration/test_v2_catalog_installed.py env: RUN_V2_CATALOG_COMPATIBILITY: "1" - name: Test v2 metadata publication and resource routes - run: uv run coverage run -a --branch -m pytest -q tests/integration/test_v2_catalog_publication.py tests/integration/test_v2_metadata_routes.py + run: uv run --frozen --extra dev coverage run -a --branch -m pytest -q tests/integration/test_v2_catalog_publication.py tests/integration/test_v2_metadata_routes.py - name: Test v2 policy persistence and immediate v1 mirroring - run: uv run coverage run -a --branch -m pytest -q tests/integration/test_v2_policy_persistence.py tests/integration/test_v1_policy_dual_write.py tests/integration/test_mysql_policy_dual_write.py tests/integration/test_v2_user_policy_mirroring.py tests/integration/test_v1_user_policy_dual_write.py + run: uv run --frozen --extra dev coverage run -a --branch -m pytest -q tests/integration/test_v2_policy_persistence.py tests/integration/test_v1_policy_dual_write.py tests/integration/test_mysql_policy_dual_write.py tests/integration/test_v2_user_policy_mirroring.py tests/integration/test_v1_user_policy_dual_write.py - name: Test signed-in identity provisioning and user associations - run: uv run coverage run -a --branch -m pytest -q tests/integration/test_v2_user_identity_persistence.py + run: uv run --frozen --extra dev coverage run -a --branch -m pytest -q tests/integration/test_v2_user_identity_persistence.py - name: Test comparison-run persistence and privilege reconciliation as the managed roles - run: uv run coverage run -a --branch -m pytest -q tests/integration/test_v2_stage12_comparison_persistence.py tests/integration/test_v2_privilege_reconciliation.py + run: uv run --frozen --extra dev coverage run -a --branch -m pytest -q tests/integration/test_v2_stage12_comparison_persistence.py tests/integration/test_v2_privilege_reconciliation.py - name: Test v2 household persistence and immediate v1 copying - run: uv run coverage run -a --branch -m pytest -q tests/integration/test_v2_household_persistence.py tests/integration/test_v1_household_dual_write.py tests/integration/test_mysql_household_spm_dual_write.py + run: uv run --frozen --extra dev coverage run -a --branch -m pytest -q tests/integration/test_v2_household_persistence.py tests/integration/test_v1_household_dual_write.py tests/integration/test_mysql_household_spm_dual_write.py - name: Qualify production-scale v2 metadata publication - run: uv run coverage run -a --branch -m pytest -q tests/integration/test_v2_catalog_publication_qualification.py + run: uv run --frozen --extra dev coverage run -a --branch -m pytest -q tests/integration/test_v2_catalog_publication_qualification.py env: RUN_V2_CATALOG_PUBLICATION_QUALIFICATION: "1" - name: Test real Redis cross-instance semantics - run: uv run coverage run -a --branch -m pytest -q tests/integration/test_runtime_cache_redis.py + run: uv run --frozen --extra dev coverage run -a --branch -m pytest -q tests/integration/test_runtime_cache_redis.py - name: Export v2 integration coverage - run: uv run coverage xml -i -o coverage-v2.xml + run: uv run --frozen --extra dev coverage xml -i -o coverage-v2.xml - name: Upload v2 integration coverage to Codecov uses: codecov/codecov-action@v5 with: diff --git a/Makefile b/Makefile index 1e9c1a80c..17743258d 100644 --- a/Makefile +++ b/Makefile @@ -1,23 +1,28 @@ .PHONY: install setup-env debug debug-asgi test-env-vars test quality-guards typecheck-v2 debug-test format changelog +# Install and test in the environment uv.lock defines, on the Python version +# the Cloud Run image runs (gcp/cloud_run/Dockerfile). .python-version selects +# 3.14 for other uv commands; set UV_PYTHON to test another version. +export UV_PYTHON ?= 3.12 + install: - pip install -e ".[dev]" + uv sync --frozen --extra dev setup-env: bash .github/setup_env.sh debug: - FLASK_APP=policyengine_api.api FLASK_DEBUG=1 flask run --without-threads + FLASK_APP=policyengine_api.api FLASK_DEBUG=1 uv run --frozen --extra dev flask run --without-threads debug-asgi: - FLASK_DEBUG=1 uvicorn policyengine_api.asgi:app --reload --port 8000 + FLASK_DEBUG=1 uv run --frozen --extra dev uvicorn policyengine_api.asgi:app --reload --port 8000 test-env-vars: - pytest tests/env_variables + uv run --frozen --extra dev pytest tests/env_variables test: - MAX_HOUSEHOLDS=1000 python -m coverage run -a --branch -m pytest tests/to_refactor tests/unit tests/contract tests/integration/test_budget_window_in_flight_dedupe.py --disable-pytest-warnings - python -m coverage xml -i + MAX_HOUSEHOLDS=1000 uv run --frozen --extra dev coverage run -a --branch -m pytest tests/to_refactor tests/unit tests/contract tests/integration/test_budget_window_in_flight_dedupe.py --disable-pytest-warnings + uv run --frozen --extra dev coverage xml -i quality-guards: python scripts/run_quality_guards.py @@ -26,7 +31,7 @@ typecheck-v2: uv run --frozen --extra dev mypy debug-test: - MAX_HOUSEHOLDS=1000 FLASK_DEBUG=1 pytest -vv --durations=0 tests + MAX_HOUSEHOLDS=1000 FLASK_DEBUG=1 uv run --frozen --extra dev pytest -vv --durations=0 tests format: ruff format . @@ -34,4 +39,4 @@ format: changelog: python .github/bump_version.py uv lock - towncrier build --yes --version $$(python -c "import re; print(re.search(r'version = \"(.+?)\"', open('pyproject.toml').read()).group(1))") + uv run --frozen --extra dev towncrier build --yes --version $$(python -c "import re; print(re.search(r'version = \"(.+?)\"', open('pyproject.toml').read()).group(1))") diff --git a/README.md b/README.md index f572bcdf9..221e2a932 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,11 @@ git clone https://github.com/PolicyEngine/policyengine-api.git To contribute, clone the repository instead of forking it and then request to be added as a contributor. Create a new branch and get started! -### 2. Activate your virtual environment +### 2. Install uv + +Install [uv](https://docs.astral.sh/uv/getting-started/installation/). The +project's dependencies are pinned in `uv.lock`, the same file the Cloud Run +image installs from. ### 3. Install dependencies @@ -35,6 +39,14 @@ make install make setup-env ``` +`make install` runs `uv sync --frozen --extra dev`, which creates `.venv` with +exactly the versions in `uv.lock` on Python 3.12, the version the Cloud Run +image runs. `make test`, `make test-env-vars`, `make debug-test`, `make debug`, +`make debug-asgi`, `make typecheck-v2` and the towncrier step of +`make changelog` run through `uv run --frozen --extra dev`, so you don't need +to activate `.venv` for them. To run another command in that environment, +prefix it the same way, for example `uv run --frozen --extra dev pytest tests/unit`. + ### 3a. Configure environment variables `make setup-env` creates a local `.env` from `.env.example`. At minimum, local development expects values for: diff --git a/changelog.d/api-tests-from-lock.changed.md b/changelog.d/api-tests-from-lock.changed.md new file mode 100644 index 000000000..90f716fdd --- /dev/null +++ b/changelog.d/api-tests-from-lock.changed.md @@ -0,0 +1 @@ +CI test jobs and the Make install and test targets install from `uv.lock` with `uv sync --frozen --extra dev` and run tests through `uv run --frozen` on Python 3.12, so tests use the same dependency versions as the Cloud Run image. diff --git a/docs/engineering/skills/testing.md b/docs/engineering/skills/testing.md index 51c36817d..344e49107 100644 --- a/docs/engineering/skills/testing.md +++ b/docs/engineering/skills/testing.md @@ -2,6 +2,24 @@ Use this skill whenever adding, moving, or reviewing tests. +## Test Environment + +Tests run in the environment `uv.lock` defines, on Python 3.12, the same +versions the Cloud Run image installs (`gcp/cloud_run/Dockerfile`): + +- `make install` runs `uv sync --frozen --extra dev`, and `make test` runs + pytest and coverage through `uv run --frozen --extra dev`. The Makefile sets + `UV_PYTHON=3.12` unless it is already set, because `.python-version` selects + 3.14. +- To run other commands in that environment, prefix them with + `uv run --frozen --extra dev --python 3.12`. The commands below omit the + prefix for brevity. +- A CI job that runs tests sets `UV_PYTHON: "3.12"`, installs with + `make install` (or `uv sync --frozen --extra dev`), runs every pytest and + coverage command through `uv run --frozen`, and never installs packages with + pip. `tests/unit/test_locked_test_environments.py` checks this for every + workflow job that runs pytest or coverage. + ## Migration Test Layout - Put API migration compatibility tests under `tests/contract/`. diff --git a/pyproject.toml b/pyproject.toml index ed101271e..f63898e63 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,6 +71,7 @@ dev = [ "pytest", "pytest-snapshot", "pytest-timeout", + "pytest-xdist", "ruff>=0.9.0", "towncrier>=24.8.0", ] diff --git a/tests/unit/test_alembic_workflows.py b/tests/unit/test_alembic_workflows.py index 797c82848..6bb8c4032 100644 --- a/tests/unit/test_alembic_workflows.py +++ b/tests/unit/test_alembic_workflows.py @@ -170,12 +170,13 @@ def test_reusable_alembic_check_uses_only_disposable_mysql(): assert "POLICYENGINE_DB_MIGRATION_PASSWORD" not in workflow -def test_reusable_alembic_check_uses_the_installed_python_environment(): +def test_reusable_alembic_check_uses_the_locked_python_environment(): workflow = _workflow("alembic-v1-check.yml") - assert "python -m pytest" in workflow - assert "python -m alembic" in workflow - assert "uv run" not in workflow + assert "run: make install" in workflow + assert "uv run --frozen --extra dev python -m pytest" in workflow + assert workflow.count("uv run --frozen --extra dev python -m alembic") == 2 + assert "pip install" not in workflow def test_reusable_v2_alembic_check_uses_only_disposable_postgres(): @@ -478,3 +479,19 @@ def test_v1_and_future_v2_alembic_domains_are_explicitly_separate(): assert "migrations/v1" in text assert "Supabase/Postgres" in text assert "separate revision chain" in text + + +def test_changelog_builds_with_the_locked_towncrier(): + workflow = _workflow("push.yml") + versioning_job = workflow[workflow.index(" versioning:") :] + versioning_job = versioning_job[: versioning_job.index("\n publish-git-tag:")] + makefile = (REPO / "Makefile").read_text(encoding="utf-8") + changelog_target = makefile[makefile.index("changelog:") :] + build_script = (REPO / ".github/scripts/build_changelog.sh").read_text( + encoding="utf-8" + ) + + assert "pip install" not in versioning_job + assert 'UV_PYTHON: "3.12"' in versioning_job + assert "uv run --frozen --extra dev towncrier build" in build_script + assert "uv run --frozen --extra dev towncrier build" in changelog_target diff --git a/tests/unit/test_cloud_run_deploy_scripts.py b/tests/unit/test_cloud_run_deploy_scripts.py index ee7a48b9f..aff99f039 100644 --- a/tests/unit/test_cloud_run_deploy_scripts.py +++ b/tests/unit/test_cloud_run_deploy_scripts.py @@ -1875,14 +1875,15 @@ def _assert_live_tests_run_in_parallel_after_readiness( assert "strategy:" not in parallel_live_tests assert "matrix:" not in parallel_live_tests assert "needs: deploy-cloud-run-staging" in parallel_live_tests - assert "pip install pytest pytest-xdist httpx" in parallel_live_tests + assert "pip install" not in parallel_live_tests + assert "run: make install" in parallel_live_tests live_readiness_command = ( 'bash .github/scripts/health_check.sh "' '${{ needs.deploy-cloud-run-staging.outputs.url }}/readiness-check"' ) assert live_readiness_command in parallel_live_tests assert parallel_live_tests.index( - "Install live test dependencies" + "Install locked live test dependencies" ) < parallel_live_tests.index(live_readiness_command) assert parallel_live_tests.index( live_readiness_command diff --git a/tests/unit/test_deploy_pipeline_scripts.py b/tests/unit/test_deploy_pipeline_scripts.py index cef35fe3d..956b09f9c 100644 --- a/tests/unit/test_deploy_pipeline_scripts.py +++ b/tests/unit/test_deploy_pipeline_scripts.py @@ -53,7 +53,8 @@ def _changelog_bin(tmp_path: Path) -> tuple[Path, Path]: bin_dir.mkdir() (bin_dir / "python").symlink_to(sys.executable) log = tmp_path / "calls.log" - _recording_command(bin_dir, "towncrier", log) + # towncrier runs from uv.lock through uv run, so uv is the command to fake. + _recording_command(bin_dir, "uv", log) return bin_dir, log @@ -68,7 +69,9 @@ def test_build_changelog_uses_the_pyproject_version(tmp_path: Path) -> None: ) assert result.returncode == 0, result.stderr - assert _lines(log) == ["towncrier build --yes --version 3.4.5"] + assert _lines(log) == [ + "uv run --frozen --extra dev towncrier build --yes --version 3.4.5" + ] def test_build_changelog_fails_without_a_version(tmp_path: Path) -> None: diff --git a/tests/unit/test_locked_test_environments.py b/tests/unit/test_locked_test_environments.py new file mode 100644 index 000000000..7224b37e1 --- /dev/null +++ b/tests/unit/test_locked_test_environments.py @@ -0,0 +1,206 @@ +"""CI test jobs install from uv.lock and run their tests through it. + +The Cloud Run image installs with ``uv sync --frozen`` from uv.lock +(gcp/cloud_run/Dockerfile). A test job that installs with pip instead resolves +the newest releases on PyPI, so its tests run against versions production does +not run. +""" + +from __future__ import annotations + +import re +from pathlib import Path +from typing import NamedTuple + +import pytest + +from scripts.rules.workflows import parse_jobs + +REPO = Path(__file__).resolve().parents[2] +WORKFLOW_DIRECTORY = REPO / ".github" / "workflows" +LOCKED_SYNC = "uv sync --frozen --extra dev" +LOCKED_RUN = "uv run --frozen --extra dev" +# The Cloud Run image runs on python:3.12-slim; .python-version selects 3.14. +PRODUCTION_PYTHON_SETTING = 'UV_PYTHON: "3.12"' +SCRIPT_REFERENCE = re.compile(r"\.github/[\w./-]+\.sh") +MAKE_TARGET_REFERENCE = re.compile(r"\bmake ([\w-]+)") +TEST_COMMAND = re.compile(r"\bpytest\b|\bcoverage (run|xml)\b") +MAKE_TARGET_HEADER = re.compile(r"^([\w-]+)\s*:(?!=)") +# The test jobs this module must find; a smaller set means the detection broke. +EXPECTED_TEST_JOBS = { + ("alembic-v1-check.yml", "mysql-lifecycle"), + ("alembic-v2-check.yml", "postgres-lifecycle"), + ("pr.yml", "test"), + ("pr.yml", "test_env_vars"), + ("push.yml", "deploy-cloud-run-candidate"), + ("push.yml", "deploy-cloud-run-staging"), + ("push.yml", "parallel-live-simulation-tests-staging-cloud-run"), + ("v2-integration-check.yml", "catalog-runtime-integration"), +} +# Operational jobs that run application code against real databases with the +# interpreter setup-python provides. They keep an explicit unlocked install +# until moving them to uv.lock is verified on its own. +UNLOCKED_OPERATIONAL_JOBS = { + ("migrate-v1-cloud-sql.yml", "migrate"), + ("process-v1-household-mirror-event.yml", "process-event"), +} +EXPLICIT_UNLOCKED_INSTALL = 'pip install -e ".[dev]"' + + +class WorkflowJob(NamedTuple): + """One workflow job and every command line it runs.""" + + workflow: str + job_id: str + job_text: str + executed_lines: list[str] + + +def _makefile_recipes() -> dict[str, list[str]]: + recipes: dict[str, list[str]] = {} + current_target = None + for makefile_line in (REPO / "Makefile").read_text(encoding="utf-8").splitlines(): + header = MAKE_TARGET_HEADER.match(makefile_line) + if header is not None and header.group(1) != ".PHONY": + current_target = header.group(1) + recipes[current_target] = [] + elif makefile_line.startswith("\t") and current_target is not None: + recipes[current_target].append(makefile_line.strip()) + elif makefile_line.strip(): + current_target = None + return recipes + + +def _executed_lines(command_text: str, visited: set[str] | None = None) -> list[str]: + """Return the command's lines plus those of the scripts and Make targets it runs.""" + visited = set() if visited is None else visited + recipes = _makefile_recipes() + executed = [] + for command_line in command_text.splitlines(): + if command_line.lstrip().startswith("#"): + continue + executed.append(command_line) + for script_path in SCRIPT_REFERENCE.findall(command_line): + if script_path in visited or not (REPO / script_path).is_file(): + continue + visited.add(script_path) + script_text = (REPO / script_path).read_text(encoding="utf-8") + executed += _executed_lines(script_text, visited) + for target in MAKE_TARGET_REFERENCE.findall(command_line): + if f"make {target}" in visited or target not in recipes: + continue + visited.add(f"make {target}") + executed += _executed_lines("\n".join(recipes[target]), visited) + return executed + + +def _job_text(workflow_text: str, job_id: str) -> str: + match = re.search( + rf"^ {re.escape(job_id)}:\n(?P.*?)(?=^ [\w-]+:\n|\Z)", + workflow_text, + flags=re.MULTILINE | re.DOTALL, + ) + assert match is not None, f"missing workflow job {job_id}" + return match.group("body") + + +def _workflow_jobs() -> list[WorkflowJob]: + """Return every workflow job with the command lines it runs.""" + jobs = [] + for workflow_path in sorted(WORKFLOW_DIRECTORY.glob("*.y*ml")): + workflow_text = workflow_path.read_text(encoding="utf-8") + parsed_jobs, problems = parse_jobs(workflow_text) + assert problems == [], f"{workflow_path.name}: {problems}" + for job_id, job in parsed_jobs.items(): + executed = [] + for run_block in job.run_blocks: + executed += _executed_lines(run_block.text) + jobs.append( + WorkflowJob( + workflow_path.name, + job_id, + _job_text(workflow_text, job_id), + executed, + ) + ) + return jobs + + +def _test_jobs() -> list[WorkflowJob]: + return [ + job + for job in _workflow_jobs() + if any( + TEST_COMMAND.search(executed_line) for executed_line in job.executed_lines + ) + ] + + +def _test_job_id(job: WorkflowJob) -> str: + return f"{job.workflow}:{job.job_id}" + + +def test_detects_every_known_test_job(): + found = {(job.workflow, job.job_id) for job in _test_jobs()} + + assert found >= EXPECTED_TEST_JOBS + + +def test_make_install_syncs_the_locked_development_environment(): + assert _makefile_recipes()["install"] == [LOCKED_SYNC] + + +@pytest.mark.parametrize("target", ["test", "test-env-vars", "debug-test"]) +def test_make_test_targets_run_through_the_locked_environment(target): + recipe = _makefile_recipes()[target] + test_commands = [ + recipe_line for recipe_line in recipe if TEST_COMMAND.search(recipe_line) + ] + + assert test_commands + assert all(LOCKED_RUN in recipe_line for recipe_line in test_commands), recipe + + +def test_makefile_runs_the_production_python_version(): + makefile = (REPO / "Makefile").read_text(encoding="utf-8") + + assert re.search(r"^export UV_PYTHON \?= 3\.12$", makefile, flags=re.MULTILINE) + + +@pytest.mark.parametrize("job", _test_jobs(), ids=_test_job_id) +def test_test_job_installs_nothing_outside_the_lock(job): + unlocked_installs = [ + executed_line + for executed_line in job.executed_lines + if "pip install" in executed_line + ] + + assert unlocked_installs == [] + + +@pytest.mark.parametrize("job", _test_jobs(), ids=_test_job_id) +def test_test_job_runs_every_test_command_through_the_lock(job): + unlocked_test_commands = [ + executed_line + for executed_line in job.executed_lines + if TEST_COMMAND.search(executed_line) and "uv run --frozen" not in executed_line + ] + + assert unlocked_test_commands == [] + + +@pytest.mark.parametrize("job", _test_jobs(), ids=_test_job_id) +def test_test_job_selects_the_production_python_version(job): + assert PRODUCTION_PYTHON_SETTING in job.job_text + + +@pytest.mark.parametrize( + ("workflow", "job_id"), sorted(UNLOCKED_OPERATIONAL_JOBS), ids=str +) +def test_operational_job_keeps_an_explicit_install_for_its_python(workflow, job_id): + jobs = {(job.workflow, job.job_id): job for job in _workflow_jobs()} + executed = jobs[(workflow, job_id)].executed_lines + + # make install now syncs .venv, which the job's bare python never reads. + assert EXPLICIT_UNLOCKED_INSTALL in executed + assert "make install" not in executed diff --git a/uv.lock b/uv.lock index 52ec2d144..8e1ed4be8 100644 --- a/uv.lock +++ b/uv.lock @@ -749,6 +749,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa", size = 18059, upload-time = "2024-10-25T17:25:39.051Z" }, ] +[[package]] +name = "execnet" +version = "2.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622, upload-time = "2025-11-12T09:56:37.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" }, +] + [[package]] name = "executing" version = "2.2.1" @@ -2819,6 +2828,7 @@ dev = [ { name = "pytest" }, { name = "pytest-snapshot" }, { name = "pytest-timeout" }, + { name = "pytest-xdist" }, { name = "ruff" }, { name = "towncrier" }, ] @@ -2858,6 +2868,7 @@ requires-dist = [ { name = "pytest", marker = "extra == 'dev'" }, { name = "pytest-snapshot", marker = "extra == 'dev'" }, { name = "pytest-timeout", marker = "extra == 'dev'" }, + { name = "pytest-xdist", marker = "extra == 'dev'" }, { name = "python-dotenv" }, { name = "redis" }, { name = "rq" }, @@ -3551,6 +3562,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl", hash = "sha256:c42667e5cdadb151aeb5b26d114aff6bdf5a907f176a007a30b940d3d865b5c2", size = 14382, upload-time = "2025-05-05T19:44:33.502Z" }, ] +[[package]] +name = "pytest-xdist" +version = "3.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "execnet" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0"