Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion .github/scripts/build_changelog.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,4 +16,6 @@ print(match.group(1))
PY
)"

towncrier build --yes --version "${version}"
# towncrier comes from the dev extra in uv.lock, which uv lock has just
# refreshed with the new project version.
uv run --frozen --extra dev towncrier build --yes --version "${version}"
4 changes: 3 additions & 1 deletion .github/scripts/run_live_staging_tests.sh
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
#!/usr/bin/env bash
set -euo pipefail

python -m pytest \
# Run from the environment uv.lock defines (make install), which includes
# pytest-xdist for the two workers.
uv run --frozen --extra dev python -m pytest \
-n 2 \
--dist load \
--maxschedchunk=1 \
Expand Down
2 changes: 1 addition & 1 deletion .github/scripts/test_alembic_v2_lifecycle.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@

set -euo pipefail

uv run pytest -q \
uv run --frozen --extra dev pytest -q \
tests/unit/v2/test_alembic_v2.py \
tests/integration/test_alembic_v2_lifecycle.py
14 changes: 10 additions & 4 deletions .github/workflows/alembic-v1-check.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@ jobs:
mysql-lifecycle:
name: Alembic MySQL lifecycle
runs-on: ubuntu-latest
env:
# uv installs uv.lock on the Cloud Run image's Python, not the 3.14
# that .python-version selects.
UV_PYTHON: "3.12"
services:
mysql:
image: mysql:8.4
Expand All @@ -30,17 +34,19 @@ jobs:
uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Install dependencies
- name: Setup uv
uses: astral-sh/setup-uv@v6
- name: Install locked dependencies
run: make install
- name: Test Alembic configuration and MySQL lifecycle
run: python -m pytest tests/unit/data/test_alembic_baseline.py tests/unit/data/test_v1_database_migration.py tests/integration/test_alembic_mysql_lifecycle.py -q
run: uv run --frozen --extra dev python -m pytest tests/unit/data/test_alembic_baseline.py tests/unit/data/test_v1_database_migration.py tests/integration/test_alembic_mysql_lifecycle.py -q
env:
ALEMBIC_DATABASE_URL: mysql+pymysql://root:policyengine_test@127.0.0.1:3306/policyengine_alembic_test
- name: Require database at all v1 heads
run: python -m alembic -c alembic-v1.ini current --check-heads
run: uv run --frozen --extra dev python -m alembic -c alembic-v1.ini current --check-heads
env:
ALEMBIC_DATABASE_URL: mysql+pymysql://root:policyengine_test@127.0.0.1:3306/policyengine_alembic_test
- name: Require no ungenerated v1 operations
run: python -m alembic -c alembic-v1.ini check
run: uv run --frozen --extra dev python -m alembic -c alembic-v1.ini check
env:
ALEMBIC_DATABASE_URL: mysql+pymysql://root:policyengine_test@127.0.0.1:3306/policyengine_alembic_test
9 changes: 6 additions & 3 deletions .github/workflows/alembic-v2-check.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@ jobs:
--health-timeout=5s
--health-retries=20
env:
# uv installs uv.lock on the Cloud Run image's Python, not the 3.14
# that .python-version selects.
UV_PYTHON: "3.12"
V2_MIGRATION_DATABASE_URL: postgresql+psycopg://postgres:policyengine_v2_test@127.0.0.1:5432/policyengine_v2_alembic_test
V2_ALEMBIC_DISPOSABLE_TEST: "1"
steps:
Expand All @@ -35,10 +38,10 @@ jobs:
- name: Setup uv
uses: astral-sh/setup-uv@v6
- name: Install locked dependencies
run: uv sync --frozen
run: uv sync --frozen --extra dev
- name: Test generated-only v2 migration configuration and lifecycle
run: bash .github/scripts/test_alembic_v2_lifecycle.sh
- name: Require database at the v2 head
run: uv run alembic -c alembic-v2.ini current --check-heads
run: uv run --frozen --extra dev alembic -c alembic-v2.ini current --check-heads
- name: Require no ungenerated v2 schema or data operations
run: uv run alembic -c alembic-v2.ini check
run: uv run --frozen --extra dev alembic -c alembic-v2.ini check
7 changes: 5 additions & 2 deletions .github/workflows/migrate-v1-cloud-sql.yml
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,11 @@ jobs:
service_account: ${{ vars.GCP_DB_MIGRATION_SERVICE_ACCOUNT }}
- name: Set up GCloud
uses: google-github-actions/setup-gcloud@v2
- name: Install dependencies
run: make install
# make install syncs .venv from uv.lock, which the bare python this
# job's scripts run does not read. Moving this job to uv.lock is a
# separate, verified change; until then it installs unlocked.
- name: Install dependencies (unlocked)
run: pip install -e ".[dev]"
- name: Start Cloud SQL Auth Proxy
run: bash .github/scripts/start_cloud_sql_proxy.sh
- name: Upgrade and verify v1 database
Expand Down
18 changes: 15 additions & 3 deletions .github/workflows/pr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -190,24 +190,30 @@ jobs:
permissions:
contents: read
id-token: write
env:
# uv installs uv.lock on the Cloud Run image's Python, not the 3.14
# that .python-version selects.
UV_PYTHON: "3.12"
steps:
- name: Checkout repo
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Set up uv
uses: astral-sh/setup-uv@v6
- name: Auth
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ secrets.GCP_DEPLOY_SERVICE_ACCOUNT }}
- name: Wait until PolicyEngine bundle version is available on PyPI
run: .github/wait-for-pypi.sh
- name: Install dependencies
- name: Install locked dependencies
run: make install
- name: Run environment variable tests
run: pytest tests/env_variables/test_environment_variables.py
run: uv run --frozen --extra dev pytest tests/env_variables/test_environment_variables.py
env:
POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }}
# Compatibility name required by PolicyEngine Core. The GitHub
Expand All @@ -223,13 +229,19 @@ jobs:
permissions:
contents: read
id-token: write
env:
# uv installs uv.lock on the Cloud Run image's Python, not the 3.14
# that .python-version selects.
UV_PYTHON: "3.12"
steps:
- name: Checkout repo
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Set up uv
uses: astral-sh/setup-uv@v6
- name: Update package lists
run: sudo apt-get update
- name: Install system dependencies
Expand All @@ -245,7 +257,7 @@ jobs:
uses: google-github-actions/setup-gcloud@v2
with:
project_id: policyengine-api
- name: Install dependencies
- name: Install locked dependencies
run: make install
- name: Test the API
run: make test
Expand Down
7 changes: 5 additions & 2 deletions .github/workflows/process-v1-household-mirror-event.yml
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,11 @@ jobs:
uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Install dependencies
run: make install
# make install syncs .venv from uv.lock, which the bare python this
# job's scripts run does not read. Moving this job to uv.lock is a
# separate, verified change; until then it installs unlocked.
- name: Install dependencies (unlocked)
run: pip install -e ".[dev]"
- name: Authenticate authorized database operator
uses: google-github-actions/auth@v2
with:
Expand Down
40 changes: 31 additions & 9 deletions .github/workflows/push.yml
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,10 @@ jobs:
(github.repository == 'PolicyEngine/policyengine-api')
&& !(github.event.head_commit.message == 'Update PolicyEngine API')
runs-on: ubuntu-latest
env:
# uv installs uv.lock on the Cloud Run image's Python, not the 3.14
# that .python-version selects.
UV_PYTHON: "3.12"
steps:
- name: Generate GitHub App token
id: app-token
Expand All @@ -115,8 +119,6 @@ jobs:
uses: astral-sh/setup-uv@v6
with:
version: "0.13.4"
- name: Install changelog tool
run: pip install towncrier
- name: Bump project version
run: python .github/bump_version.py
- name: Refresh uv lock
Expand Down Expand Up @@ -229,6 +231,9 @@ jobs:
CLOUD_RUN_MIN_INSTANCES: "0"
CLOUD_RUN_SERVICE_MIN_INSTANCES: "0"
CLOUD_RUN_MAX_INSTANCES: "1"
# uv installs uv.lock on the Cloud Run image's Python, not the 3.14
# that .python-version selects.
UV_PYTHON: "3.12"
permissions:
contents: read
id-token: write
Expand All @@ -246,13 +251,15 @@ jobs:
uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Setup uv
uses: astral-sh/setup-uv@v6
- name: Update package lists
run: sudo apt-get update
- name: Install release test dependencies
run: sudo apt-get install -y redis-server
- name: Start release test Redis
run: sudo systemctl start redis-server
- name: Install application dependencies
- name: Install locked application dependencies
run: make install
- name: Run release tests
run: bash .github/scripts/run_release_tests.sh
Expand Down Expand Up @@ -322,7 +329,7 @@ jobs:
- name: Wait for Cloud Run staging health
run: bash .github/scripts/health_check.sh "${{ steps.candidate.outputs.url }}/readiness-check"
- name: Run staging integration tests
run: python -m pytest tests/integration/test_cloud_run_candidate.py tests/integration/test_live_v2_metadata.py tests/integration/test_live_v2_policies.py -v
run: uv run --frozen --extra dev python -m pytest tests/integration/test_cloud_run_candidate.py tests/integration/test_live_v2_metadata.py tests/integration/test_live_v2_policies.py -v
env:
API_BASE_URL: ${{ steps.candidate.outputs.url }}
STAGING_API_TEST_PROBE_ID: cloud-run-${{ steps.cloud_run.outputs.revision_tag }}
Expand All @@ -344,15 +351,21 @@ jobs:
if: |
(github.repository == 'PolicyEngine/policyengine-api')
&& (github.event.head_commit.message == 'Update PolicyEngine API')
env:
# uv installs uv.lock on the Cloud Run image's Python, not the 3.14
# that .python-version selects.
UV_PYTHON: "3.12"
steps:
- name: Checkout repo
uses: actions/checkout@v4
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Install live test dependencies
run: pip install pytest pytest-xdist httpx
- name: Setup uv
uses: astral-sh/setup-uv@v6
- name: Install locked live test dependencies
run: make install
- name: Wait for Cloud Run staging candidate health before live tests
run: bash .github/scripts/health_check.sh "${{ needs.deploy-cloud-run-staging.outputs.url }}/readiness-check"
- name: Run live staging tests in two pytest workers
Expand Down Expand Up @@ -524,12 +537,21 @@ jobs:
# daytime peak, so full traffic needs the extra headroom.
CLOUD_RUN_SERVICE_MIN_INSTANCES: "2"
CLOUD_RUN_MAX_INSTANCES: "8"
# uv installs uv.lock on the Cloud Run image's Python, not the 3.14
# that .python-version selects.
UV_PYTHON: "3.12"
permissions:
contents: read
id-token: write
steps:
- name: Checkout repo
uses: actions/checkout@v4
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Setup uv
uses: astral-sh/setup-uv@v6
- name: GCP authentication
uses: "google-github-actions/auth@v2"
with:
Expand Down Expand Up @@ -575,10 +597,10 @@ jobs:
CLOUD_RUN_TAG: ${{ steps.cloud_run.outputs.revision_tag }}
- name: Wait for Cloud Run candidate health
run: bash .github/scripts/health_check.sh "${{ steps.candidate.outputs.url }}/readiness-check"
- name: Install Cloud Run smoke test dependencies
run: pip install pytest httpx
- name: Install locked Cloud Run smoke test dependencies
run: make install
- name: Run Cloud Run candidate smoke tests
run: python -m pytest tests/integration/test_cloud_run_candidate.py tests/integration/test_live_v2_metadata.py -v
run: uv run --frozen --extra dev python -m pytest tests/integration/test_cloud_run_candidate.py tests/integration/test_live_v2_metadata.py -v
env:
API_BASE_URL: ${{ steps.candidate.outputs.url }}
STAGING_API_TEST_PROBE_ID: cloud-run-${{ steps.cloud_run.outputs.revision_tag }}
Expand Down
31 changes: 17 additions & 14 deletions .github/workflows/v2-integration-check.yml
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,9 @@ jobs:
--health-timeout=5s
--health-retries=20
env:
# uv installs uv.lock on the Cloud Run image's Python, not the 3.14
# that .python-version selects.
UV_PYTHON: "3.12"
ALEMBIC_DATABASE_URL: mysql+pymysql://root:policyengine_test@127.0.0.1:3306/policyengine_alembic_test
V2_MIGRATION_DATABASE_URL: postgresql+psycopg://postgres:policyengine_v2_test@127.0.0.1:5432/policyengine_v2_alembic_test
V2_ALEMBIC_DISPOSABLE_TEST: "1"
Expand All @@ -64,37 +67,37 @@ jobs:
- name: Setup uv
uses: astral-sh/setup-uv@v6
- name: Install locked dependencies
run: uv sync --frozen
run: uv sync --frozen --extra dev
- name: Prepare the disposable v1 schema
run: uv run alembic -c alembic-v1.ini upgrade head
run: uv run --frozen --extra dev alembic -c alembic-v1.ini upgrade head
- name: Prepare the disposable v2 schema
run: uv run alembic -c alembic-v2.ini upgrade head
run: uv run --frozen --extra dev alembic -c alembic-v2.ini upgrade head
- name: Create the managed v2 roles and grant their declared privileges
run: uv run python scripts/reconcile_v2_privileges.py apply --create-missing-roles
run: uv run --frozen --extra dev python scripts/reconcile_v2_privileges.py apply --create-missing-roles
- name: Verify the managed v2 roles hold their declared privileges
run: uv run python scripts/reconcile_v2_privileges.py verify
run: uv run --frozen --extra dev python scripts/reconcile_v2_privileges.py verify
- name: Verify the installed PolicyEngine.py catalog interface
run: uv run coverage run --branch -m pytest -q tests/integration/test_v2_catalog_installed.py
run: uv run --frozen --extra dev coverage run --branch -m pytest -q tests/integration/test_v2_catalog_installed.py
env:
RUN_V2_CATALOG_COMPATIBILITY: "1"
- name: Test v2 metadata publication and resource routes
run: uv run coverage run -a --branch -m pytest -q tests/integration/test_v2_catalog_publication.py tests/integration/test_v2_metadata_routes.py
run: uv run --frozen --extra dev coverage run -a --branch -m pytest -q tests/integration/test_v2_catalog_publication.py tests/integration/test_v2_metadata_routes.py
- name: Test v2 policy persistence and immediate v1 mirroring
run: uv run coverage run -a --branch -m pytest -q tests/integration/test_v2_policy_persistence.py tests/integration/test_v1_policy_dual_write.py tests/integration/test_mysql_policy_dual_write.py tests/integration/test_v2_user_policy_mirroring.py tests/integration/test_v1_user_policy_dual_write.py
run: uv run --frozen --extra dev coverage run -a --branch -m pytest -q tests/integration/test_v2_policy_persistence.py tests/integration/test_v1_policy_dual_write.py tests/integration/test_mysql_policy_dual_write.py tests/integration/test_v2_user_policy_mirroring.py tests/integration/test_v1_user_policy_dual_write.py
- name: Test signed-in identity provisioning and user associations
run: uv run coverage run -a --branch -m pytest -q tests/integration/test_v2_user_identity_persistence.py
run: uv run --frozen --extra dev coverage run -a --branch -m pytest -q tests/integration/test_v2_user_identity_persistence.py
- name: Test comparison-run persistence and privilege reconciliation as the managed roles
run: uv run coverage run -a --branch -m pytest -q tests/integration/test_v2_stage12_comparison_persistence.py tests/integration/test_v2_privilege_reconciliation.py
run: uv run --frozen --extra dev coverage run -a --branch -m pytest -q tests/integration/test_v2_stage12_comparison_persistence.py tests/integration/test_v2_privilege_reconciliation.py
- name: Test v2 household persistence and immediate v1 copying
run: uv run coverage run -a --branch -m pytest -q tests/integration/test_v2_household_persistence.py tests/integration/test_v1_household_dual_write.py tests/integration/test_mysql_household_spm_dual_write.py
run: uv run --frozen --extra dev coverage run -a --branch -m pytest -q tests/integration/test_v2_household_persistence.py tests/integration/test_v1_household_dual_write.py tests/integration/test_mysql_household_spm_dual_write.py
- name: Qualify production-scale v2 metadata publication
run: uv run coverage run -a --branch -m pytest -q tests/integration/test_v2_catalog_publication_qualification.py
run: uv run --frozen --extra dev coverage run -a --branch -m pytest -q tests/integration/test_v2_catalog_publication_qualification.py
env:
RUN_V2_CATALOG_PUBLICATION_QUALIFICATION: "1"
- name: Test real Redis cross-instance semantics
run: uv run coverage run -a --branch -m pytest -q tests/integration/test_runtime_cache_redis.py
run: uv run --frozen --extra dev coverage run -a --branch -m pytest -q tests/integration/test_runtime_cache_redis.py
- name: Export v2 integration coverage
run: uv run coverage xml -i -o coverage-v2.xml
run: uv run --frozen --extra dev coverage xml -i -o coverage-v2.xml
- name: Upload v2 integration coverage to Codecov
uses: codecov/codecov-action@v5
with:
Expand Down
Loading
Loading