diff --git a/changelog_entry.yaml b/changelog_entry.yaml index 2e68ba388..caad276de 100644 --- a/changelog_entry.yaml +++ b/changelog_entry.yaml @@ -16,6 +16,7 @@ - Ran Stage 12 US national simulations as 20 region-group calls each, and built an independent, immutable Stage 12 cache of three annual US datasets and 60 current-law baselines before each deployment, publishing it only after a recorded determinism check and recording its release in the v2 version manifest under a new key, while the existing key keeps a copy that earlier readers accept - Made the budget refuse a household count outside its country's plausible range (above 200 million in the US or 60 million in the UK, negative or not finite), raising ImplausibleEntityCountError instead of publishing a count like #767's changed: + - Moved the entrypoint's backend calls, response telemetry and comparison dispatch out of app.py (925 lines, now 583) into entry_runtime.py, which app.py re-exports and which still logs under the policyengine_simulation_entry.app logger; the routes stay in app.py, so FastAPI and ruff read their annotations as before; the 2,251-line deployment asset test file is split into seven test files and a support module named test_deployment_assets_support.py, the rules registry names each moved test by its new file, and its guard-test discovery now covers tests/test_deployment_assets*.py - Moved the worker's credential setup, request year and policy parsing, and region resolution out of simulation_runtime.py (1,111 lines, now 651) into gcp_credentials.py, request_policy.py and region_resolution.py; simulation_runtime.py keeps dataset selection and loading, simulation building, local-area breakdowns and the run itself, which tests replace by name there, and re-exports the moved names, so every caller still imports them from simulation_runtime; the shared helpers _parse_year, _normalise_policy, _compile_interval_policy, _runtime_policy (now wrapper_policy), _resolve_region, _resolve_dataset_reference and _require_local_area_breakdown are now public, and the credential messages still log under the simulation_runtime logger; the 3,048-line output builder test file is split into seven test files and a module of shared fakes, all named test_simulation_output_*.py so the weighted-frame rule still checks them - Split the artifact precompute (precompute.py, 931 lines) and the Stage 12 cache precompute (stage12_cache/precompute.py, 844 lines) each into planning, dataset-building and baseline modules, with the manifest and run_precompute left in the original module, which re-exports every public name, so the Modal precompute apps and their tests import from the same paths; the cache precompute's _scoped_planned_simulation and _baseline_identity are now public, and the worker validation imports scoped_planned_simulation from its new module; the 1,835-line precompute test file is split into four, and the cache precompute tests patch each name in the module whose code reads it - Split the 4,709-line Stage 12 runtime test file into thirteen test files by behaviour (dataset loading, provenance, the single worker and its failures, aggregates, the coordinator and its failures, runtime store retries, failure records, cancellation, child state errors and the wait window) and three support modules for the shared builders, fake stores and fake child calls, whose helpers are now public; the other Stage 12 tests import them from those modules, every test keeps its name and body, and two parametrized cases are named after the renamed bundle helper diff --git a/docs/generated/rules.md b/docs/generated/rules.md index 4fc82bdaf..917f0df80 100644 --- a/docs/generated/rules.md +++ b/docs/generated/rules.md @@ -66,7 +66,7 @@ Every rule that governs `policyengine-sim-api` is listed here with how it is enf - `sim.ci.multiline-run-is-script`: 13 remaining in `rules/baselines/ci-multiline-run.txt` - `sim.microdata.weight-stripping-confined`: 0 remaining in `rules/baselines/sim-microdata-weight-stripping.txt` -- `sim.quality.python-file-length`: 2 remaining in `rules/baselines/python-file-length.txt` +- `sim.quality.python-file-length`: 0 remaining in `rules/baselines/python-file-length.txt` ### Exceptions and review dates @@ -102,20 +102,21 @@ flowchart LR | Rule | Statement | Tier | Status | Blocking | Enforced by | CI | Proof | | --- | --- | --- | --- | --- | --- | --- | --- | -| `sim.stage12.deployment-scope` | Main-branch deploys roll out Stage 12 in both environments; a Stage 12-only deploy runs candidate tests and the requested promotion but never redeploys existing Modal resources or runs the existing stack's complete integration suite; the Stage 12 worker can request a Google OIDC token. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | +| `sim.stage12.deployment-scope` | Main-branch deploys roll out Stage 12 in both environments; a Stage 12-only deploy runs candidate tests and the requested promotion but never redeploys existing Modal resources or runs the existing stack's complete integration suite; the Stage 12 worker can request a Google OIDC token. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets_promotion.py`, `projects/policyengine-simulation-entry/tests/test_deployment_assets_release_gates.py` | `pr.yml:test` | **unproven** | | `sim.stage12.deployment-uses-api-schema` | Stage 12 deployment uses the API-owned schema directly, and the Stage 12 runtime it deploys uses typed DML without raw SQL or schema changes. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | | `sim.stage12.no-migration-chain` | This repository has no schema migration chain (no Alembic); Stage 12 tables are migrated by policyengine-api. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py` | `pr.yml:test` | **unproven** | | `sim.stage12.no-production-user-associations` | Stage 12 runtime modules never create production user associations (user_simulations or user_reports). | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py` | `pr.yml:test` | **unproven** | | `sim.stage12.no-schema-ddl` | Simulation runtimes and the Stage 12 persistence library contain no schema DDL (CREATE, ALTER, DROP or TRUNCATE of tables, types, schemas or databases, or create_all); policyengine-api owns the schema. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py` | `pr.yml:test` | yes | | `sim.stage12.persistence-typed-dml-only` | The Stage 12 persistence library and runtime use typed SQLAlchemy Core DML only: no raw SQL text, driver-level SQL or schema operations. | mechanical | enforced | yes | pytest `libs/policyengine-stage12-persistence/tests/test_source_boundaries.py` | `pr.yml:test` | **unproven** | -| `sim.stage12.production-lock-has-db-runtime` | The production lock file includes the direct Stage 12 database runtime dependencies. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | +| `sim.stage12.production-lock-has-db-runtime` | The production lock file includes the direct Stage 12 database runtime dependencies. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets_traffic.py` | `pr.yml:test` | **unproven** | ```mermaid flowchart LR r_sim_stage12_deployment_scope("sim.stage12.deployment-scope (enforced)") - e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py["pytest projects/policyengine-simulation-entry/tests/test_deployment_assets.py"] + e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_promotion_py__projects_policyengine_simulation_entry_tests_test_deployment_assets_release_gates_py["pytest projects/policyengine-simulation-entry/tests/test_deployment_assets_promotion.py, projects/policyengine-simulation-entry/tests/test_deployment_assets_release_gates.py"] j_CI_pr_yml_test["CI pr.yml:test"] r_sim_stage12_deployment_uses_api_schema("sim.stage12.deployment-uses-api-schema (enforced)") + e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py["pytest projects/policyengine-simulation-entry/tests/test_deployment_assets.py"] r_sim_stage12_no_migration_chain("sim.stage12.no-migration-chain (enforced)") e_pytest_projects_policyengine_simulation_entry_tests_test_stage12_schema_authority_py["pytest projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py"] r_sim_stage12_no_production_user_associations("sim.stage12.no-production-user-associations (enforced)") @@ -123,88 +124,105 @@ flowchart LR r_sim_stage12_persistence_typed_dml_only("sim.stage12.persistence-typed-dml-only (enforced)") e_pytest_libs_policyengine_stage12_persistence_tests_test_source_boundaries_py["pytest libs/policyengine-stage12-persistence/tests/test_source_boundaries.py"] r_sim_stage12_production_lock_has_db_runtime("sim.stage12.production-lock-has-db-runtime (enforced)") - r_sim_stage12_deployment_scope --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py - e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py --> j_CI_pr_yml_test + e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_traffic_py["pytest projects/policyengine-simulation-entry/tests/test_deployment_assets_traffic.py"] + r_sim_stage12_deployment_scope --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_promotion_py__projects_policyengine_simulation_entry_tests_test_deployment_assets_release_gates_py + e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_promotion_py__projects_policyengine_simulation_entry_tests_test_deployment_assets_release_gates_py --> j_CI_pr_yml_test r_sim_stage12_deployment_uses_api_schema --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py + e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py --> j_CI_pr_yml_test r_sim_stage12_no_migration_chain --> e_pytest_projects_policyengine_simulation_entry_tests_test_stage12_schema_authority_py e_pytest_projects_policyengine_simulation_entry_tests_test_stage12_schema_authority_py --> j_CI_pr_yml_test r_sim_stage12_no_production_user_associations --> e_pytest_projects_policyengine_simulation_entry_tests_test_stage12_schema_authority_py r_sim_stage12_no_schema_ddl --> e_pytest_projects_policyengine_simulation_entry_tests_test_stage12_schema_authority_py r_sim_stage12_persistence_typed_dml_only --> e_pytest_libs_policyengine_stage12_persistence_tests_test_source_boundaries_py e_pytest_libs_policyengine_stage12_persistence_tests_test_source_boundaries_py --> j_CI_pr_yml_test - r_sim_stage12_production_lock_has_db_runtime --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py + r_sim_stage12_production_lock_has_db_runtime --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_traffic_py + e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_traffic_py --> j_CI_pr_yml_test ``` ## Deployment pipeline | Rule | Statement | Tier | Status | Blocking | Enforced by | CI | Proof | | --- | --- | --- | --- | --- | --- | --- | --- | -| `sim.deploy.container-context-minimal` | Container builds exclude local environments and unrelated projects, and the entrypoint container builds for the requested target platform. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | +| `sim.deploy.container-context-minimal` | Container builds exclude local environments and unrelated projects, and the entrypoint container builds for the requested target platform. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets_cloud_run.py`, `projects/policyengine-simulation-entry/tests/test_deployment_assets_traffic.py` | `pr.yml:test` | **unproven** | | `sim.deploy.docker-images-build` | The simulation entry and executor Docker images build. | mechanical | enforced | yes | CI job `docker-build` | `pr.yml:docker-build` | — | | `sim.deploy.gcloud-without-terraform` | Deployment uses the gcloud-based workflow, not Terraform. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | | `sim.deploy.image-smoke-covers-every-image-lib` | The image smoke path filter includes every library mounted into a Modal image, so a change to any of them runs the smoke. | mechanical | doc-only | no | nothing | — | — | | `sim.deploy.modal-image-smoke` | App entrypoints import inside the real Modal images before merge, whenever a library baked into an image changes. | mechanical | enforced | yes | CI job `image-smoke` | `pr-image-smoke.yml:image-smoke` | — | -| `sim.deploy.numerical-sentinel-gate` | An existing-stack deployment computes the numerical release sentinel from the exact bundle it installs, and the candidate and stable services must reproduce it; a failing sentinel check fails its step. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | -| `sim.deploy.permission-enforcement-wired` | Simulation Entrypoint permission enforcement defaults to log-only, rejects any value other than 0 or 1, and reaches the candidate revision and every deployed authentication run. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | -| `sim.deploy.promotion-order-explicit` | Full-stack promotion order is explicit, and the reusable deployment rejects unexpectedly skipped phases. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | -| `sim.deploy.release-identity-smoke` | Deploy gates pass the whole release identity to the entrypoint smoke, which requires every expectation, refuses malformed ones, and rejects a release whose routes, latest pointers, bundle, UK model or UK dataset differ from that identity. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | -| `sim.deploy.revision-tags-valid` | Cloud Run revision tags are valid on a workflow's first run and fit the existing service names. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | -| `sim.deploy.routing-publication-journaled` | Each routing publication is named by its run, attempt and environment and journaled before it is applied; an always-running finalizer resolves it from the journal with the release-gate outcome, a release reports success only after that finalizer completes, and routing jobs never tolerate a failed command. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | +| `sim.deploy.numerical-sentinel-gate` | An existing-stack deployment computes the numerical release sentinel from the exact bundle it installs, and the candidate and stable services must reproduce it; a failing sentinel check fails its step. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets_promotion.py`, `projects/policyengine-simulation-entry/tests/test_deployment_assets_release_gates.py` | `pr.yml:test` | **unproven** | +| `sim.deploy.permission-enforcement-wired` | Simulation Entrypoint permission enforcement defaults to log-only, rejects any value other than 0 or 1, and reaches the candidate revision and every deployed authentication run. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets_cloud_run.py` | `pr.yml:test` | **unproven** | +| `sim.deploy.promotion-order-explicit` | Full-stack promotion order is explicit, and the reusable deployment rejects unexpectedly skipped phases. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets_promotion.py` | `pr.yml:test` | **unproven** | +| `sim.deploy.release-identity-smoke` | Deploy gates pass the whole release identity to the entrypoint smoke, which requires every expectation, refuses malformed ones, and rejects a release whose routes, latest pointers, bundle, UK model or UK dataset differ from that identity. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets_identity_smoke.py`, `projects/policyengine-simulation-entry/tests/test_deployment_assets_release_gates.py`, `projects/policyengine-simulation-entry/tests/test_deployment_assets_routing.py` | `pr.yml:test` | **unproven** | +| `sim.deploy.revision-tags-valid` | Cloud Run revision tags are valid on a workflow's first run and fit the existing service names. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets_cloud_run.py` | `pr.yml:test` | **unproven** | +| `sim.deploy.routing-publication-journaled` | Each routing publication is named by its run, attempt and environment and journaled before it is applied; an always-running finalizer resolves it from the journal with the release-gate outcome, a release reports success only after that finalizer completes, and routing jobs never tolerate a failed command. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets_release_gates.py`, `projects/policyengine-simulation-entry/tests/test_deployment_assets_routing.py` | `pr.yml:test` | **unproven** | | `sim.deploy.shell-scripts-valid` | Deployment shell scripts have valid shell syntax. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | | `sim.deploy.smoke-tests-strict` | Candidate and stable smoke tests reject a revision mismatch without retrying, and fail if the previous revision does not converge. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | -| `sim.deploy.traffic-changes-exact` | Traffic changes target an exact, validated revision, are reversible, and refuse to run over an intervening promotion. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | -| `sim.deploy.workflow-shell-safety` | Cloud Run deployment never interpolates GitHub expressions into shell, escapes environment values, and pins secret versions. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | +| `sim.deploy.traffic-changes-exact` | Traffic changes target an exact, validated revision, are reversible, and refuse to run over an intervening promotion. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets_traffic.py` | `pr.yml:test` | **unproven** | +| `sim.deploy.workflow-shell-safety` | Cloud Run deployment never interpolates GitHub expressions into shell, escapes environment values, and pins secret versions. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets_cloud_run.py` | `pr.yml:test` | **unproven** | ```mermaid flowchart LR r_sim_deploy_container_context_minimal("sim.deploy.container-context-minimal (enforced)") - e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py["pytest projects/policyengine-simulation-entry/tests/test_deployment_assets.py"] + e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_cloud_run_py__projects_policyengine_simulation_entry_tests_test_deployment_assets_traffic_py["pytest projects/policyengine-simulation-entry/tests/test_deployment_assets_cloud_run.py, projects/policyengine-simulation-entry/tests/test_deployment_assets_traffic.py"] j_CI_pr_yml_test["CI pr.yml:test"] r_sim_deploy_docker_images_build("sim.deploy.docker-images-build (enforced)") e_CI_job_docker_build["CI job docker-build"] j_CI_pr_yml_docker_build["CI pr.yml:docker-build"] r_sim_deploy_gcloud_without_terraform("sim.deploy.gcloud-without-terraform (enforced)") + e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py["pytest projects/policyengine-simulation-entry/tests/test_deployment_assets.py"] r_sim_deploy_image_smoke_covers_every_image_lib("sim.deploy.image-smoke-covers-every-image-lib (doc-only)") gap_unenforced>"not enforced"] r_sim_deploy_modal_image_smoke("sim.deploy.modal-image-smoke (enforced)") e_CI_job_image_smoke["CI job image-smoke"] j_CI_pr_image_smoke_yml_image_smoke["CI pr-image-smoke.yml:image-smoke"] r_sim_deploy_numerical_sentinel_gate("sim.deploy.numerical-sentinel-gate (enforced)") + e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_promotion_py__projects_policyengine_simulation_entry_tests_test_deployment_assets_release_gates_py["pytest projects/policyengine-simulation-entry/tests/test_deployment_assets_promotion.py, projects/policyengine-simulation-entry/tests/test_deployment_assets_release_gates.py"] r_sim_deploy_permission_enforcement_wired("sim.deploy.permission-enforcement-wired (enforced)") + e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_cloud_run_py["pytest projects/policyengine-simulation-entry/tests/test_deployment_assets_cloud_run.py"] r_sim_deploy_promotion_order_explicit("sim.deploy.promotion-order-explicit (enforced)") + e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_promotion_py["pytest projects/policyengine-simulation-entry/tests/test_deployment_assets_promotion.py"] r_sim_deploy_release_identity_smoke("sim.deploy.release-identity-smoke (enforced)") + e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_identity_smoke_py__projects_policyengine_simulation_entry_tests_test_deployment_assets_release_gates_py__projects_policyengine_simulation_entry_tests_test_deployment_assets_routing_py["pytest projects/policyengine-simulation-entry/tests/test_deployment_assets_identity_smoke.py, projects/policyengine-simulation-entry/tests/test_deployment_assets_release_gates.py, projects/policyengine-simulation-entry/tests/test_deployment_assets_routing.py"] r_sim_deploy_revision_tags_valid("sim.deploy.revision-tags-valid (enforced)") r_sim_deploy_routing_publication_journaled("sim.deploy.routing-publication-journaled (enforced)") + e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_release_gates_py__projects_policyengine_simulation_entry_tests_test_deployment_assets_routing_py["pytest projects/policyengine-simulation-entry/tests/test_deployment_assets_release_gates.py, projects/policyengine-simulation-entry/tests/test_deployment_assets_routing.py"] r_sim_deploy_shell_scripts_valid("sim.deploy.shell-scripts-valid (enforced)") r_sim_deploy_smoke_tests_strict("sim.deploy.smoke-tests-strict (enforced)") r_sim_deploy_traffic_changes_exact("sim.deploy.traffic-changes-exact (enforced)") + e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_traffic_py["pytest projects/policyengine-simulation-entry/tests/test_deployment_assets_traffic.py"] r_sim_deploy_workflow_shell_safety("sim.deploy.workflow-shell-safety (enforced)") - r_sim_deploy_container_context_minimal --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py - e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py --> j_CI_pr_yml_test + r_sim_deploy_container_context_minimal --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_cloud_run_py__projects_policyengine_simulation_entry_tests_test_deployment_assets_traffic_py + e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_cloud_run_py__projects_policyengine_simulation_entry_tests_test_deployment_assets_traffic_py --> j_CI_pr_yml_test r_sim_deploy_docker_images_build --> e_CI_job_docker_build e_CI_job_docker_build --> j_CI_pr_yml_docker_build r_sim_deploy_gcloud_without_terraform --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py + e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py --> j_CI_pr_yml_test r_sim_deploy_image_smoke_covers_every_image_lib -.-> gap_unenforced r_sim_deploy_modal_image_smoke --> e_CI_job_image_smoke e_CI_job_image_smoke --> j_CI_pr_image_smoke_yml_image_smoke - r_sim_deploy_numerical_sentinel_gate --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py - r_sim_deploy_permission_enforcement_wired --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py - r_sim_deploy_promotion_order_explicit --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py - r_sim_deploy_release_identity_smoke --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py - r_sim_deploy_revision_tags_valid --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py - r_sim_deploy_routing_publication_journaled --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py + r_sim_deploy_numerical_sentinel_gate --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_promotion_py__projects_policyengine_simulation_entry_tests_test_deployment_assets_release_gates_py + e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_promotion_py__projects_policyengine_simulation_entry_tests_test_deployment_assets_release_gates_py --> j_CI_pr_yml_test + r_sim_deploy_permission_enforcement_wired --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_cloud_run_py + e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_cloud_run_py --> j_CI_pr_yml_test + r_sim_deploy_promotion_order_explicit --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_promotion_py + e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_promotion_py --> j_CI_pr_yml_test + r_sim_deploy_release_identity_smoke --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_identity_smoke_py__projects_policyengine_simulation_entry_tests_test_deployment_assets_release_gates_py__projects_policyengine_simulation_entry_tests_test_deployment_assets_routing_py + e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_identity_smoke_py__projects_policyengine_simulation_entry_tests_test_deployment_assets_release_gates_py__projects_policyengine_simulation_entry_tests_test_deployment_assets_routing_py --> j_CI_pr_yml_test + r_sim_deploy_revision_tags_valid --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_cloud_run_py + r_sim_deploy_routing_publication_journaled --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_release_gates_py__projects_policyengine_simulation_entry_tests_test_deployment_assets_routing_py + e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_release_gates_py__projects_policyengine_simulation_entry_tests_test_deployment_assets_routing_py --> j_CI_pr_yml_test r_sim_deploy_shell_scripts_valid --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py r_sim_deploy_smoke_tests_strict --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py - r_sim_deploy_traffic_changes_exact --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py - r_sim_deploy_workflow_shell_safety --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py + r_sim_deploy_traffic_changes_exact --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_traffic_py + e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_traffic_py --> j_CI_pr_yml_test + r_sim_deploy_workflow_shell_safety --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_cloud_run_py ``` ## API contracts and integration | Rule | Statement | Tier | Status | Blocking | Enforced by | CI | Proof | | --- | --- | --- | --- | --- | --- | --- | --- | -| `sim.contracts.clients-publish-deployed-commit` | Client publication checks out the commit that was actually deployed. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | -| `sim.contracts.full-integration-beta-only` | The complete integration suite runs against beta deployments only, never production. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | +| `sim.contracts.clients-publish-deployed-commit` | Client publication checks out the commit that was actually deployed. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets_traffic.py` | `pr.yml:test` | **unproven** | +| `sim.contracts.full-integration-beta-only` | The complete integration suite runs against beta deployments only, never production. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets_promotion.py` | `pr.yml:test` | **unproven** | | `sim.contracts.gateway-openapi-golden` | The gateway's OpenAPI spec matches the checked-in golden file; regenerate it deliberately when the API changes. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-gateway/tests/test_openapi_golden.py` | `pr.yml:test` | **unproven** | | `sim.contracts.local-integration` | Generated-client integration tests pass against the locally built services. | mechanical | enforced | yes | CI job `test-integration` | `pr.yml:test-integration` | — | | `sim.contracts.regenerate-clients` | When public API schemas change, regenerate the clients and run the generated-client integration tests. | mechanical | doc-only | no | nothing | — | — | @@ -212,9 +230,10 @@ flowchart LR ```mermaid flowchart LR r_sim_contracts_clients_publish_deployed_commit("sim.contracts.clients-publish-deployed-commit (enforced)") - e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py["pytest projects/policyengine-simulation-entry/tests/test_deployment_assets.py"] + e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_traffic_py["pytest projects/policyengine-simulation-entry/tests/test_deployment_assets_traffic.py"] j_CI_pr_yml_test["CI pr.yml:test"] r_sim_contracts_full_integration_beta_only("sim.contracts.full-integration-beta-only (enforced)") + e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_promotion_py["pytest projects/policyengine-simulation-entry/tests/test_deployment_assets_promotion.py"] r_sim_contracts_gateway_openapi_golden("sim.contracts.gateway-openapi-golden (enforced)") e_pytest_projects_policyengine_simulation_gateway_tests_test_openapi_golden_py["pytest projects/policyengine-simulation-gateway/tests/test_openapi_golden.py"] r_sim_contracts_local_integration("sim.contracts.local-integration (enforced)") @@ -222,9 +241,10 @@ flowchart LR j_CI_pr_yml_test_integration["CI pr.yml:test-integration"] r_sim_contracts_regenerate_clients("sim.contracts.regenerate-clients (doc-only)") gap_unenforced>"not enforced"] - r_sim_contracts_clients_publish_deployed_commit --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py - e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py --> j_CI_pr_yml_test - r_sim_contracts_full_integration_beta_only --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py + r_sim_contracts_clients_publish_deployed_commit --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_traffic_py + e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_traffic_py --> j_CI_pr_yml_test + r_sim_contracts_full_integration_beta_only --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_promotion_py + e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_promotion_py --> j_CI_pr_yml_test r_sim_contracts_gateway_openapi_golden --> e_pytest_projects_policyengine_simulation_gateway_tests_test_openapi_golden_py e_pytest_projects_policyengine_simulation_gateway_tests_test_openapi_golden_py --> j_CI_pr_yml_test r_sim_contracts_local_integration --> e_CI_job_test_integration diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py index 9bb6d3af9..be3a699ce 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py @@ -1,15 +1,16 @@ -"""FastAPI application for the Cloud Run Simulation Entrypoint.""" +"""FastAPI application for the Cloud Run Simulation Entrypoint. + +The backend calls and comparison dispatch the routes use live in +entry_runtime; this module re-exports its public names. +""" from __future__ import annotations import asyncio -import contextlib import logging -import time import uuid from contextlib import asynccontextmanager -from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Literal, NamedTuple, Protocol +from typing import TYPE_CHECKING from uuid import UUID from fastapi import Depends, FastAPI, Request @@ -30,21 +31,17 @@ VersionMap, VersionsResponse, ) -from policyengine_simulation_contract.json_types import JsonObject from policyengine_simulation_contract.stage12_execution import ( ComparisonReportRecord, ComparisonRunLifecycleStatus, - ComparisonSimulationRecord, ) from policyengine_simulation_observability.identifiers import ( OBSERVABILITY_ID_HEADER, normalize_observability_id, - resolve_observability_id, ) from policyengine_simulation_observability.observability import ( init_simulation_observability, ) -from pydantic import BaseModel, TypeAdapter, ValidationError from starlette.datastructures import MutableHeaders from policyengine_simulation_entry.auth import ( @@ -53,18 +50,21 @@ CallerAuthenticator, CallerAuthorizer, ) -from policyengine_simulation_entry.backend import ( - BackendResponse, - BackendTimeoutError, - BackendUnavailableError, - OldGatewayBackend, - SimulationBackend, -) +from policyengine_simulation_entry.backend import OldGatewayBackend, SimulationBackend from policyengine_simulation_entry.config import Settings +from policyengine_simulation_entry.entry_runtime import ( + BACKEND_RESPONSE_HEADER, + STAGE12_MODAL_SUBMISSION_TIMEOUT_SECONDS, + BackendCall, + ComparisonBackend, + EntryRuntime, + ResponseIdentifier, + model_json, + request_identifiers, + route_template, + set_observability_header, +) from policyengine_simulation_entry.schemas import ( - BackendTelemetryAttributes, - BackendTelemetryPayload, - RequestIdentifiers, TemporaryStage12ReportResponse, TemporaryStage12SubmissionResponse, ) @@ -77,170 +77,18 @@ if TYPE_CHECKING: from collections.abc import AsyncIterator - from policyengine_observability import ObservabilityRuntime from starlette.middleware.base import RequestResponseEndpoint -logger = logging.getLogger(__name__) -BACKEND_RESPONSE_HEADER = { - "X-PolicyEngine-Simulation-Backend": "old_gateway", -} -_json_object_adapter = TypeAdapter(JsonObject) -type ResponseIdentifier = Literal["job_id", "batch_job_id"] -STAGE12_MODAL_SUBMISSION_TIMEOUT_SECONDS = 5.0 - - -class ComparisonBackend(Protocol): - """Stage 12 comparison dispatch and polling seam used by the entry routes. - - The real implementation is ``Stage12ComparisonBackend``; tests inject fakes. - The routes bound every call with a timeout and never let a failure here - alter the production v1 response. - """ - - async def dispatch_after_production( - self, - *, - request_payload: dict[str, Any], - production_response: bytes, - request_id: str, - observability_id: str, - ) -> None: - """Start a shadow comparison run for a production job the gateway accepted. - - ``production_response`` is the raw gateway response body; its ``job_id`` - identifies the incumbent run being compared. - """ - ... - - async def submit_temporary_report( - self, - *, - request_payload: dict[str, Any], - request_id: str, - observability_id: str, - ) -> ComparisonReportRecord: - """Submit an operator-requested report and return its acknowledged record.""" - ... - - async def get_temporary_report( - self, - evaluation_id: UUID, - ) -> tuple[ComparisonReportRecord, tuple[ComparisonSimulationRecord, ...]]: - """Return a report and its simulations; raise LookupError if it is unknown.""" - ... - - -def _model_json(model: BaseModel) -> JsonObject: - payload = model.model_dump(mode="json", by_alias=True, exclude_none=True) - selection = getattr(model, "spm", None) - if selection is not None: - # An explicit null information date clears a bundle cutoff. Preserve - # that distinction while leaving omitted request fields omitted. - payload["spm"] = selection.model_dump(mode="json", exclude_unset=True) - return _json_object_adapter.validate_python(payload) - - -def _response(result: BackendResponse) -> Response: - headers = { - **result.headers, - **BACKEND_RESPONSE_HEADER, - } - return Response( - content=result.content, - status_code=result.status_code, - headers=headers, - ) - - -def _route_template(request: Request) -> str: - return getattr(request.scope.get("route"), "path", request.url.path) - - -def _request_identifiers(request: Request) -> RequestIdentifiers: - identifiers: RequestIdentifiers = {} - job_id = request.path_params.get("job_id") - batch_job_id = request.path_params.get("batch_job_id") - evaluation_id = request.path_params.get("evaluation_id") - if isinstance(job_id, str): - identifiers["job_id"] = job_id - if isinstance(batch_job_id, str): - identifiers["batch_job_id"] = batch_job_id - if isinstance(evaluation_id, str): - identifiers["evaluation_id"] = evaluation_id - return identifiers - - -@dataclass(frozen=True) -class _BackendCall: - """One request a route forwards to the old gateway, and what its event records. - - ``identifiers`` come from the route's path; ``response_identifier_key`` - names the response field to copy into the backend response event. - """ - - method: str - path: str - body: JsonObject | None = None - identifiers: RequestIdentifiers | None = None - response_identifier_key: ResponseIdentifier | None = None - - -class _BackendFailure(NamedTuple): - """The event and sanitized response for an old-gateway call that failed.""" - - event_name: str - status_code: int - detail: str - +__all__ = [ + "BACKEND_RESPONSE_HEADER", + "STAGE12_MODAL_SUBMISSION_TIMEOUT_SECONDS", + "ComparisonBackend", + "ResponseIdentifier", + "app", + "create_app", +] -_BACKEND_TIMEOUT = _BackendFailure( - "simulation_entry_backend_timeout", 504, "Simulation backend timed out." -) -_BACKEND_UNAVAILABLE = _BackendFailure( - "simulation_entry_backend_unavailable", 503, "Simulation backend is unavailable." -) - - -def _backend_observability_id(headers: dict[str, str]) -> str | None: - """Return the backend's normalized observability ID header, if it sent one.""" - return next( - ( - normalize_observability_id(value) - for key, value in headers.items() - if key.lower() == OBSERVABILITY_ID_HEADER.lower() - ), - None, - ) - - -def _set_observability_header(response: Response, observability_id: str | None) -> None: - """Return ``observability_id`` on the response, or drop any stale header.""" - if observability_id is not None: - response.headers[OBSERVABILITY_ID_HEADER] = observability_id - elif OBSERVABILITY_ID_HEADER in response.headers: - del response.headers[OBSERVABILITY_ID_HEADER] - - -def _add_response_telemetry( - attributes: BackendTelemetryAttributes, - content: bytes, - response_identifier_key: ResponseIdentifier | None, -) -> None: - """Add the job state and requested identifier from a JSON backend response.""" - try: - response_payload = BackendTelemetryPayload.model_validate_json(content) - except ValidationError: - response_payload = None - job_state = response_payload.status if response_payload is not None else None - if job_state is not None: - attributes["job_state"] = job_state - response_identifier = ( - getattr(response_payload, response_identifier_key) - if response_payload is not None and response_identifier_key - else None - ) - if response_identifier_key and response_identifier is not None: - attributes[response_identifier_key] = response_identifier +logger = logging.getLogger(__name__) def _temporary_submission_payload( @@ -263,196 +111,6 @@ def _stage12_unavailable_response(detail: str) -> JSONResponse: ) -class _EntryRuntime: - """The settings, telemetry runtime and backends the entry routes share.""" - - def __init__( - self, - *, - settings: Settings, - runtime: ObservabilityRuntime, - backend: SimulationBackend, - comparison: ComparisonBackend | None, - ) -> None: - """Keep the dependencies that ``create_app`` resolved.""" - self.settings = settings - self.runtime = runtime - self.backend = backend - self.comparison = comparison - - def bind_observability_id(self, request: Request, value: object) -> str | None: - """Bind a normalized observability ID to the request and telemetry context.""" - observability_id = normalize_observability_id(value) - request.state.observability_id = observability_id - if observability_id is not None: - # Telemetry is non-fatal. - with contextlib.suppress(Exception): - self.runtime.set_context(observability_id=observability_id) - return observability_id - - def start_workflow_observability(self, request: Request) -> str: - """Bind the caller's observability ID, or a new one, to a workflow request.""" - observability_id = resolve_observability_id( - getattr(request.state, "incoming_observability_id", None) - ) - self.bind_observability_id(request, observability_id) - return observability_id - - def adopt_backend_observability_id( - self, - request: Request, - headers: dict[str, str], - ) -> str | None: - """Keep the entry's observability ID, else adopt the backend's.""" - current = normalize_observability_id(request.state.observability_id) - downstream = _backend_observability_id(headers) - if current is not None and downstream is not None and current != downstream: - # Telemetry is non-fatal. - with contextlib.suppress(Exception): - self.runtime.event( - "simulation_entry_observability_id_mismatch", - attributes={ - "backend": "old_gateway", - "reason": "downstream_identifier_differed", - }, - ) - return current - return self.bind_observability_id(request, current or downstream) - - async def forward(self, request: Request, call: _BackendCall) -> Response: - """Forward ``call`` to the old gateway and relay its response. - - Timeouts and an unavailable gateway become sanitized 504 and 503 - responses; every outcome is reported as a telemetry event. - """ - backend_started = time.monotonic() - route = _route_template(request) - event_identifiers: RequestIdentifiers = call.identifiers or {} - # Telemetry is non-fatal. - with contextlib.suppress(Exception): - self.runtime.set_context(backend="old_gateway", **event_identifiers) - try: - result = await self.backend.request( - call.method, - call.path, - json_body=call.body, - request_id=request.state.request_id, - observability_id=request.state.observability_id, - ) - observability_id = self.adopt_backend_observability_id( - request, - result.headers, - ) - attributes: BackendTelemetryAttributes = { - "request_id": request.state.request_id, - "route": route, - "method": call.method, - "status_code": result.status_code, - "elapsed_ms": round( - (time.monotonic() - backend_started) * 1000, - 2, - ), - **event_identifiers, - } - _add_response_telemetry( - attributes, result.content, call.response_identifier_key - ) - self.runtime.event( - "simulation_entry_backend_response", - attributes=attributes, - ) - response = _response(result) - _set_observability_header(response, observability_id) - except BackendTimeoutError: - return self._backend_failure_response( - request, _BACKEND_TIMEOUT, route=route, identifiers=event_identifiers - ) - except BackendUnavailableError: - return self._backend_failure_response( - request, - _BACKEND_UNAVAILABLE, - route=route, - identifiers=event_identifiers, - ) - else: - return response - - def _backend_failure_response( - self, - request: Request, - failure: _BackendFailure, - *, - route: str, - identifiers: RequestIdentifiers, - ) -> JSONResponse: - """Report a failed old-gateway call and return its sanitized response.""" - self.runtime.event( - failure.event_name, - attributes={ - "request_id": request.state.request_id, - "route": route, - **identifiers, - }, - ) - return JSONResponse( - status_code=failure.status_code, - content={"detail": failure.detail}, - headers={ - "Retry-After": "10", - **BACKEND_RESPONSE_HEADER, - }, - ) - - async def dispatch_comparison_run( - self, - request: Request, - request_payload: JsonObject, - response: Response, - observability_id: str, - ) -> None: - """Start a Stage 12 comparison of an accepted submission, if enabled. - - Waits only for Modal to acknowledge the already-deployed report - coordinator; a timeout or failure is logged with no request data and - never changes the production response. - """ - comparison = self.comparison - if not ( - self.settings.stage12_enabled - and comparison is not None - and response.status_code in {200, 202} - ): - return - try: - await asyncio.wait_for( - comparison.dispatch_after_production( - request_payload=request_payload, - production_response=bytes(response.body), - request_id=request.state.request_id, - observability_id=observability_id, - ), - timeout=STAGE12_MODAL_SUBMISSION_TIMEOUT_SECONDS, - ) - except TimeoutError: - logger.error( # noqa: TRY400 # an expected timeout, fully described by timeout_seconds; the traceback would show only asyncio cancellation - "stage12_comparison_dispatch_timed_out", - extra={ - "request_id": request.state.request_id, - "timeout_seconds": STAGE12_MODAL_SUBMISSION_TIMEOUT_SECONDS, - }, - ) - # Automatic Stage 12 work must never change the production v1 - # response, including for unexpected integration failures. - except Exception as error: # noqa: BLE001 # Stage 12 dispatch must never change the v1 response - logger.error( # noqa: TRY400 # Stage 12 failure logs carry only the error type; a traceback could expose request or calculation data - "stage12_comparison_dispatch_failed", - extra={ - "request_id": request.state.request_id, - "error_type": type(error).__name__, - }, - ) - - def _install_request_context(app: FastAPI, settings: Settings) -> None: """Add the middleware that assigns request IDs and correlation headers.""" @@ -477,8 +135,8 @@ async def request_context( extra={ "request_id": request_id, "method": request.method, - "path": _route_template(request), - **_request_identifiers(request), + "path": route_template(request), + **request_identifiers(request), }, ) response = Response( @@ -487,7 +145,7 @@ async def request_context( media_type="text/plain", headers={REQUEST_ID_HEADER: request_id}, ) - _set_observability_header( + set_observability_header( response, normalize_observability_id(request.state.observability_id) ) if settings.revision: @@ -497,7 +155,7 @@ async def request_context( def _register_temporary_stage12_submission( app: FastAPI, - entry: _EntryRuntime, + entry: EntryRuntime, stage12_operator: AuthorizationDependency, ) -> None: """Add the operator-only route that submits a temporary Stage 12 report.""" @@ -525,7 +183,7 @@ async def submit_temporary_stage12_report( try: report = await asyncio.wait_for( comparison.submit_temporary_report( - request_payload=_model_json(body), + request_payload=model_json(body), request_id=request.state.request_id, observability_id=observability_id, ), @@ -576,7 +234,7 @@ async def submit_temporary_stage12_report( def _register_temporary_stage12_status( app: FastAPI, - entry: _EntryRuntime, + entry: EntryRuntime, stage12_operator: AuthorizationDependency, ) -> None: """Add the operator-only route that reads a temporary Stage 12 report.""" @@ -647,7 +305,7 @@ async def get_temporary_stage12_report( def _register_simulation_routes( app: FastAPI, - entry: _EntryRuntime, + entry: EntryRuntime, *, submitter: AuthorizationDependency, reader: AuthorizationDependency, @@ -679,10 +337,10 @@ async def submit_comparison( request: Request, ) -> Response: observability_id = entry.start_workflow_observability(request) - request_payload = _model_json(body) + request_payload = model_json(body) response = await entry.forward( request, - _BackendCall( + BackendCall( "POST", "/simulate/economy/comparison", request_payload, @@ -720,10 +378,10 @@ async def submit_budget_window( entry.start_workflow_observability(request) return await entry.forward( request, - _BackendCall( + BackendCall( "POST", "/simulate/economy/budget-window", - _model_json(body), + model_json(body), response_identifier_key="batch_job_id", ), ) @@ -757,7 +415,7 @@ async def submit_budget_window( async def get_job(job_id: str, request: Request) -> Response: return await entry.forward( request, - _BackendCall("GET", f"/jobs/{job_id}", identifiers={"job_id": job_id}), + BackendCall("GET", f"/jobs/{job_id}", identifiers={"job_id": job_id}), ) @app.get( @@ -790,7 +448,7 @@ async def get_budget_window_job( ) -> Response: return await entry.forward( request, - _BackendCall( + BackendCall( "GET", f"/budget-window-jobs/{batch_job_id}", identifiers={"batch_job_id": batch_job_id}, @@ -798,7 +456,7 @@ async def get_budget_window_job( ) -def _register_service_routes(app: FastAPI, entry: _EntryRuntime) -> None: +def _register_service_routes(app: FastAPI, entry: EntryRuntime) -> None: """Add the public version, health, readiness and ping routes.""" @app.get( @@ -809,7 +467,7 @@ def _register_service_routes(app: FastAPI, entry: _EntryRuntime) -> None: response_model=VersionsResponse, ) async def list_versions(request: Request) -> Response: - return await entry.forward(request, _BackendCall("GET", "/versions")) + return await entry.forward(request, BackendCall("GET", "/versions")) @app.get( "/versions/{kind}", @@ -819,7 +477,7 @@ async def list_versions(request: Request) -> Response: response_model=VersionMap, ) async def get_country_versions(kind: str, request: Request) -> Response: - return await entry.forward(request, _BackendCall("GET", f"/versions/{kind}")) + return await entry.forward(request, BackendCall("GET", f"/versions/{kind}")) @app.get( "/health", @@ -847,7 +505,7 @@ async def ready() -> Response: ) async def ping(body: PingRequest, request: Request) -> Response: return await entry.forward( - request, _BackendCall("POST", "/ping", _model_json(body)) + request, BackendCall("POST", "/ping", model_json(body)) ) @@ -902,7 +560,7 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: runtime_settings, runtime=runtime, ) - entry = _EntryRuntime( + entry = EntryRuntime( settings=runtime_settings, runtime=runtime, backend=runtime_backend, diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/entry_runtime.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/entry_runtime.py new file mode 100644 index 000000000..279700705 --- /dev/null +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/entry_runtime.py @@ -0,0 +1,400 @@ +"""The entrypoint's backend calls, response telemetry and comparison dispatch.""" + +from __future__ import annotations + +import asyncio +import contextlib +import logging +import time +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Literal, NamedTuple, Protocol + +from fastapi.responses import JSONResponse, Response +from policyengine_simulation_contract.json_types import JsonObject +from policyengine_simulation_observability.identifiers import ( + OBSERVABILITY_ID_HEADER, + normalize_observability_id, + resolve_observability_id, +) +from pydantic import BaseModel, TypeAdapter, ValidationError + +from policyengine_simulation_entry.backend import ( + BackendResponse, + BackendTimeoutError, + BackendUnavailableError, + SimulationBackend, +) +from policyengine_simulation_entry.schemas import ( + BackendTelemetryAttributes, + BackendTelemetryPayload, + RequestIdentifiers, +) + +if TYPE_CHECKING: + from uuid import UUID + + from fastapi import Request + from policyengine_observability import ObservabilityRuntime + from policyengine_simulation_contract.stage12_execution import ( + ComparisonReportRecord, + ComparisonSimulationRecord, + ) + + from policyengine_simulation_entry.config import Settings + +# Moved out of app.py; this code still logs under that module's name. +logger = logging.getLogger("policyengine_simulation_entry.app") + +BACKEND_RESPONSE_HEADER = { + "X-PolicyEngine-Simulation-Backend": "old_gateway", +} +_json_object_adapter = TypeAdapter(JsonObject) +type ResponseIdentifier = Literal["job_id", "batch_job_id"] +STAGE12_MODAL_SUBMISSION_TIMEOUT_SECONDS = 5.0 + + +class ComparisonBackend(Protocol): + """Stage 12 comparison dispatch and polling seam used by the entry routes. + + The real implementation is ``Stage12ComparisonBackend``; tests inject fakes. + The routes bound every call with a timeout and never let a failure here + alter the production v1 response. + """ + + async def dispatch_after_production( + self, + *, + request_payload: dict[str, Any], + production_response: bytes, + request_id: str, + observability_id: str, + ) -> None: + """Start a shadow comparison run for a production job the gateway accepted. + + ``production_response`` is the raw gateway response body; its ``job_id`` + identifies the incumbent run being compared. + """ + ... + + async def submit_temporary_report( + self, + *, + request_payload: dict[str, Any], + request_id: str, + observability_id: str, + ) -> ComparisonReportRecord: + """Submit an operator-requested report and return its acknowledged record.""" + ... + + async def get_temporary_report( + self, + evaluation_id: UUID, + ) -> tuple[ComparisonReportRecord, tuple[ComparisonSimulationRecord, ...]]: + """Return a report and its simulations; raise LookupError if it is unknown.""" + ... + + +def model_json(model: BaseModel) -> JsonObject: + """Return a request model as the JSON object the backends receive.""" + payload = model.model_dump(mode="json", by_alias=True, exclude_none=True) + selection = getattr(model, "spm", None) + if selection is not None: + # An explicit null information date clears a bundle cutoff. Preserve + # that distinction while leaving omitted request fields omitted. + payload["spm"] = selection.model_dump(mode="json", exclude_unset=True) + return _json_object_adapter.validate_python(payload) + + +def _response(result: BackendResponse) -> Response: + headers = { + **result.headers, + **BACKEND_RESPONSE_HEADER, + } + return Response( + content=result.content, + status_code=result.status_code, + headers=headers, + ) + + +def route_template(request: Request) -> str: + """Return the matched route's path template, or the request path.""" + return getattr(request.scope.get("route"), "path", request.url.path) + + +def request_identifiers(request: Request) -> RequestIdentifiers: + """Return the job, batch and evaluation IDs in the request's path.""" + identifiers: RequestIdentifiers = {} + job_id = request.path_params.get("job_id") + batch_job_id = request.path_params.get("batch_job_id") + evaluation_id = request.path_params.get("evaluation_id") + if isinstance(job_id, str): + identifiers["job_id"] = job_id + if isinstance(batch_job_id, str): + identifiers["batch_job_id"] = batch_job_id + if isinstance(evaluation_id, str): + identifiers["evaluation_id"] = evaluation_id + return identifiers + + +@dataclass(frozen=True) +class BackendCall: + """One request a route forwards to the old gateway, and what its event records. + + ``identifiers`` come from the route's path; ``response_identifier_key`` + names the response field to copy into the backend response event. + """ + + method: str + path: str + body: JsonObject | None = None + identifiers: RequestIdentifiers | None = None + response_identifier_key: ResponseIdentifier | None = None + + +class _BackendFailure(NamedTuple): + """The event and sanitized response for an old-gateway call that failed.""" + + event_name: str + status_code: int + detail: str + + +_BACKEND_TIMEOUT = _BackendFailure( + "simulation_entry_backend_timeout", 504, "Simulation backend timed out." +) +_BACKEND_UNAVAILABLE = _BackendFailure( + "simulation_entry_backend_unavailable", 503, "Simulation backend is unavailable." +) + + +def _backend_observability_id(headers: dict[str, str]) -> str | None: + """Return the backend's normalized observability ID header, if it sent one.""" + return next( + ( + normalize_observability_id(value) + for key, value in headers.items() + if key.lower() == OBSERVABILITY_ID_HEADER.lower() + ), + None, + ) + + +def set_observability_header(response: Response, observability_id: str | None) -> None: + """Return ``observability_id`` on the response, or drop any stale header.""" + if observability_id is not None: + response.headers[OBSERVABILITY_ID_HEADER] = observability_id + elif OBSERVABILITY_ID_HEADER in response.headers: + del response.headers[OBSERVABILITY_ID_HEADER] + + +def _add_response_telemetry( + attributes: BackendTelemetryAttributes, + content: bytes, + response_identifier_key: ResponseIdentifier | None, +) -> None: + """Add the job state and requested identifier from a JSON backend response.""" + try: + response_payload = BackendTelemetryPayload.model_validate_json(content) + except ValidationError: + response_payload = None + job_state = response_payload.status if response_payload is not None else None + if job_state is not None: + attributes["job_state"] = job_state + response_identifier = ( + getattr(response_payload, response_identifier_key) + if response_payload is not None and response_identifier_key + else None + ) + if response_identifier_key and response_identifier is not None: + attributes[response_identifier_key] = response_identifier + + +class EntryRuntime: + """The settings, telemetry runtime and backends the entry routes share.""" + + def __init__( + self, + *, + settings: Settings, + runtime: ObservabilityRuntime, + backend: SimulationBackend, + comparison: ComparisonBackend | None, + ) -> None: + """Keep the dependencies that ``create_app`` resolved.""" + self.settings = settings + self.runtime = runtime + self.backend = backend + self.comparison = comparison + + def bind_observability_id(self, request: Request, value: object) -> str | None: + """Bind a normalized observability ID to the request and telemetry context.""" + observability_id = normalize_observability_id(value) + request.state.observability_id = observability_id + if observability_id is not None: + # Telemetry is non-fatal. + with contextlib.suppress(Exception): + self.runtime.set_context(observability_id=observability_id) + return observability_id + + def start_workflow_observability(self, request: Request) -> str: + """Bind the caller's observability ID, or a new one, to a workflow request.""" + observability_id = resolve_observability_id( + getattr(request.state, "incoming_observability_id", None) + ) + self.bind_observability_id(request, observability_id) + return observability_id + + def adopt_backend_observability_id( + self, + request: Request, + headers: dict[str, str], + ) -> str | None: + """Keep the entry's observability ID, else adopt the backend's.""" + current = normalize_observability_id(request.state.observability_id) + downstream = _backend_observability_id(headers) + if current is not None and downstream is not None and current != downstream: + # Telemetry is non-fatal. + with contextlib.suppress(Exception): + self.runtime.event( + "simulation_entry_observability_id_mismatch", + attributes={ + "backend": "old_gateway", + "reason": "downstream_identifier_differed", + }, + ) + return current + return self.bind_observability_id(request, current or downstream) + + async def forward(self, request: Request, call: BackendCall) -> Response: + """Forward ``call`` to the old gateway and relay its response. + + Timeouts and an unavailable gateway become sanitized 504 and 503 + responses; every outcome is reported as a telemetry event. + """ + backend_started = time.monotonic() + route = route_template(request) + event_identifiers: RequestIdentifiers = call.identifiers or {} + # Telemetry is non-fatal. + with contextlib.suppress(Exception): + self.runtime.set_context(backend="old_gateway", **event_identifiers) + try: + result = await self.backend.request( + call.method, + call.path, + json_body=call.body, + request_id=request.state.request_id, + observability_id=request.state.observability_id, + ) + observability_id = self.adopt_backend_observability_id( + request, + result.headers, + ) + attributes: BackendTelemetryAttributes = { + "request_id": request.state.request_id, + "route": route, + "method": call.method, + "status_code": result.status_code, + "elapsed_ms": round( + (time.monotonic() - backend_started) * 1000, + 2, + ), + **event_identifiers, + } + _add_response_telemetry( + attributes, result.content, call.response_identifier_key + ) + self.runtime.event( + "simulation_entry_backend_response", + attributes=attributes, + ) + response = _response(result) + set_observability_header(response, observability_id) + except BackendTimeoutError: + return self._backend_failure_response( + request, _BACKEND_TIMEOUT, route=route, identifiers=event_identifiers + ) + except BackendUnavailableError: + return self._backend_failure_response( + request, + _BACKEND_UNAVAILABLE, + route=route, + identifiers=event_identifiers, + ) + else: + return response + + def _backend_failure_response( + self, + request: Request, + failure: _BackendFailure, + *, + route: str, + identifiers: RequestIdentifiers, + ) -> JSONResponse: + """Report a failed old-gateway call and return its sanitized response.""" + self.runtime.event( + failure.event_name, + attributes={ + "request_id": request.state.request_id, + "route": route, + **identifiers, + }, + ) + return JSONResponse( + status_code=failure.status_code, + content={"detail": failure.detail}, + headers={ + "Retry-After": "10", + **BACKEND_RESPONSE_HEADER, + }, + ) + + async def dispatch_comparison_run( + self, + request: Request, + request_payload: JsonObject, + response: Response, + observability_id: str, + ) -> None: + """Start a Stage 12 comparison of an accepted submission, if enabled. + + Waits only for Modal to acknowledge the already-deployed report + coordinator; a timeout or failure is logged with no request data and + never changes the production response. + """ + comparison = self.comparison + if not ( + self.settings.stage12_enabled + and comparison is not None + and response.status_code in {200, 202} + ): + return + try: + await asyncio.wait_for( + comparison.dispatch_after_production( + request_payload=request_payload, + production_response=bytes(response.body), + request_id=request.state.request_id, + observability_id=observability_id, + ), + timeout=STAGE12_MODAL_SUBMISSION_TIMEOUT_SECONDS, + ) + except TimeoutError: + logger.error( # noqa: TRY400 # an expected timeout, fully described by timeout_seconds; the traceback would show only asyncio cancellation + "stage12_comparison_dispatch_timed_out", + extra={ + "request_id": request.state.request_id, + "timeout_seconds": STAGE12_MODAL_SUBMISSION_TIMEOUT_SECONDS, + }, + ) + # Automatic Stage 12 work must never change the production v1 + # response, including for unexpected integration failures. + except Exception as error: # noqa: BLE001 # Stage 12 dispatch must never change the v1 response + logger.error( # noqa: TRY400 # Stage 12 failure logs carry only the error type; a traceback could expose request or calculation data + "stage12_comparison_dispatch_failed", + extra={ + "request_id": request.state.request_id, + "error_type": type(error).__name__, + }, + ) diff --git a/projects/policyengine-simulation-entry/tests/test_app.py b/projects/policyengine-simulation-entry/tests/test_app.py index 200ea4f41..6c0443123 100644 --- a/projects/policyengine-simulation-entry/tests/test_app.py +++ b/projects/policyengine-simulation-entry/tests/test_app.py @@ -16,6 +16,7 @@ SimulationRole, ) from policyengine_simulation_entry import app as app_module +from policyengine_simulation_entry import entry_runtime from policyengine_simulation_entry.app import create_app from policyengine_simulation_entry.backend import ( BackendResponse, @@ -278,7 +279,9 @@ async def dispatch_after_production(self, **_): "policyengine_bundle": {"model_version": "1.0.0"}, } backend.responses[("POST", "/simulate/economy/comparison")] = response(202, payload) - monkeypatch.setattr(app_module, "STAGE12_MODAL_SUBMISSION_TIMEOUT_SECONDS", 0.001) + monkeypatch.setattr( + entry_runtime, "STAGE12_MODAL_SUBMISSION_TIMEOUT_SECONDS", 0.001 + ) app = create_app( settings=automatic_stage12_settings(), backend=backend, diff --git a/projects/policyengine-simulation-entry/tests/test_deployment_assets.py b/projects/policyengine-simulation-entry/tests/test_deployment_assets.py index de151fae2..a47600cbc 100644 --- a/projects/policyengine-simulation-entry/tests/test_deployment_assets.py +++ b/projects/policyengine-simulation-entry/tests/test_deployment_assets.py @@ -1,198 +1,55 @@ +"""Tests for script syntax, the stable smoke, Stage 12 schema use and gcloud.""" + from __future__ import annotations -import json import os -import re -import subprocess -import textwrap -from pathlib import Path - -import pytest -import tomllib - -REPOSITORY_ROOT = Path(__file__).resolve().parents[3] -GITIGNORE = REPOSITORY_ROOT / ".gitignore" -TRAFFIC_SCRIPT = ( - REPOSITORY_ROOT - / ".github" - / "scripts" - / "set-cloud-run-simulation-entry-revision.sh" -) -SMOKE_SCRIPT = ( - REPOSITORY_ROOT / ".github" / "scripts" / "cloud-run-simulation-entry-smoke.sh" -) -STAGE12_VALIDATION_SCRIPT = ( - REPOSITORY_ROOT / ".github" / "scripts" / "stage12-validate-infrastructure.sh" -) -CLOUD_RUN_DEPLOY_SCRIPT = ( - REPOSITORY_ROOT / ".github" / "scripts" / "deploy-cloud-run-simulation-entry.sh" -) -DEPLOYMENT_READY_SCRIPT = ( - REPOSITORY_ROOT / ".github" / "scripts" / "simulation-deployment-ready.sh" +from typing import TYPE_CHECKING + +from test_deployment_assets_support import ( + AUTHENTICATED_TESTS, + CLOUD_RUN_DEPLOY_SCRIPT, + DEPLOY_WORKFLOW, + REPOSITORY_ROOT, + REUSABLE_DEPLOY_WORKFLOW, + SMOKE_SCRIPT, + STAGE12_DEPLOY_WORKFLOW, + STAGE12_VALIDATION_SCRIPT, + TRAFFIC_SCRIPT, + run_bash, + smoke_test_env, + write_fake_smoke_curl, ) -DEPLOY_WORKFLOW = REPOSITORY_ROOT / ".github" / "workflows" / "simulation-deploy.yml" -REUSABLE_DEPLOY_WORKFLOW = ( - REPOSITORY_ROOT / ".github" / "workflows" / "simulation-deploy.reusable.yml" -) -ROLLBACK_WORKFLOW = ( - REPOSITORY_ROOT / ".github" / "workflows" / "simulation-rollback.yml" -) -STAGE12_DEPLOY_WORKFLOW = ( - REPOSITORY_ROOT / ".github" / "workflows" / "stage12-deploy.yml" -) -PUBLISH_WORKFLOW = REPOSITORY_ROOT / ".github" / "workflows" / "publish-clients.yml" -AUTHENTICATED_TESTS = ( - REPOSITORY_ROOT - / "projects" - / "policyengine-simulation-entry" - / "authenticated_tests" -) -DOCKERIGNORE = ( - REPOSITORY_ROOT - / "projects" - / "policyengine-simulation-entry" - / "Dockerfile.dockerignore" -) -DOCKERFILE = ( - REPOSITORY_ROOT / "projects" / "policyengine-simulation-entry" / "Dockerfile" -) - -def _run_bash( - *arguments: str | Path, - check: bool, - capture_output: bool = False, - env: dict[str, str] | None = None, - cwd: Path | None = None, -) -> subprocess.CompletedProcess[str]: - """Run bash on a repository deployment script, with text output.""" - return subprocess.run( - ["bash", *arguments], - check=check, - capture_output=capture_output, - text=True, - env=env, - cwd=cwd, - ) +if TYPE_CHECKING: + from pathlib import Path def test_deployment_scripts_have_valid_shell_syntax(): - _run_bash("-n", TRAFFIC_SCRIPT, check=True) - _run_bash("-n", SMOKE_SCRIPT, check=True) - _run_bash("-n", CLOUD_RUN_DEPLOY_SCRIPT, check=True) + run_bash("-n", TRAFFIC_SCRIPT, check=True) + run_bash("-n", SMOKE_SCRIPT, check=True) + run_bash("-n", CLOUD_RUN_DEPLOY_SCRIPT, check=True) assert os.access(SMOKE_SCRIPT, os.X_OK) assert os.access(CLOUD_RUN_DEPLOY_SCRIPT, os.X_OK) -def _write_fake_smoke_curl(tmp_path: Path, revisions: list[str]) -> tuple[Path, Path]: - state_file = tmp_path / "health-request-count" - revisions_file = tmp_path / "health-revisions" - revisions_file.write_text("\n".join(revisions) + "\n", encoding="utf-8") - fake_curl = tmp_path / "curl" - fake_curl.write_text( - textwrap.dedent("""\ - #!/usr/bin/env bash - set -euo pipefail - - headers_file="" - body_file="" - url="" - while [ "$#" -gt 0 ]; do - case "$1" in - --dump-header|--output|--request|--header|--data) - option="$1" - value="$2" - shift 2 - case "${option}" in - --dump-header) headers_file="${value}" ;; - --output) body_file="${value}" ;; - esac - ;; - --fail|--silent|--show-error) - shift - ;; - *) - url="$1" - shift - ;; - esac - done - - case "${url}" in - */health) - count=0 - if [ -f "${FAKE_CURL_STATE}" ]; then - count="$(cat "${FAKE_CURL_STATE}")" - fi - line_number=$((count + 1)) - revision="$(sed -n "${line_number}p" "${FAKE_CURL_REVISIONS}")" - printf '%s' "${line_number}" > "${FAKE_CURL_STATE}" - printf 'HTTP/1.1 200 OK\r\nX-PolicyEngine-Simulation-Revision: %s\r\n\r\n' \ - "${revision}" > "${headers_file}" - printf '{"status":"healthy"}' > "${body_file}" - ;; - */ready) - printf '{"status":"ready"}' - ;; - */versions) - versions='{}' - if [ -n "${FAKE_CURL_VERSIONS:-}" ]; then - versions="$(cat "${FAKE_CURL_VERSIONS}")" - fi - if [ -n "${body_file}" ]; then - printf '%s' "${versions}" > "${body_file}" - else - printf '%s' "${versions}" - fi - ;; - */ping) - printf '{"incremented":2}' - ;; - *) - printf 'Unexpected fake curl URL: %s\n' "${url}" >&2 - exit 2 - ;; - esac - """), # noqa: E501 # the fake curl writes its HTTP header block with one printf - encoding="utf-8", - ) - fake_curl.chmod(0o755) - return state_file, revisions_file - - -def _smoke_test_env( - tmp_path: Path, - state_file: Path, - revisions_file: Path, -) -> dict[str, str]: - return { - **os.environ, - "PATH": f"{tmp_path}{os.pathsep}{os.environ['PATH']}", - "FAKE_CURL_STATE": str(state_file), - "FAKE_CURL_REVISIONS": str(revisions_file), - "SIMULATION_ENTRYPOINT_ROUTING_MAX_ATTEMPTS": "3", - "SIMULATION_ENTRYPOINT_ROUTING_RETRY_DELAY_SECONDS": "0", - } - - def test_stable_smoke_retries_the_previous_revision_until_target_is_served( tmp_path: Path, ): old_revision = "policyengine-simulation-entry-00001-old" target_revision = "policyengine-simulation-entry-00002-new" - state_file, revisions_file = _write_fake_smoke_curl( + state_file, revisions_file = write_fake_smoke_curl( tmp_path, [old_revision, old_revision, target_revision], ) - _run_bash( + run_bash( SMOKE_SCRIPT, "https://simulation.example.test", target_revision, old_revision, check=True, capture_output=True, - env=_smoke_test_env(tmp_path, state_file, revisions_file), + env=smoke_test_env(tmp_path, state_file, revisions_file), ) assert state_file.read_text(encoding="utf-8") == "3" @@ -202,19 +59,19 @@ def test_stable_smoke_rejects_an_unexpected_revision_without_retrying(tmp_path: old_revision = "policyengine-simulation-entry-00001-old" target_revision = "policyengine-simulation-entry-00002-new" unexpected_revision = "policyengine-simulation-entry-00003-unexpected" - state_file, revisions_file = _write_fake_smoke_curl( + state_file, revisions_file = write_fake_smoke_curl( tmp_path, [unexpected_revision, target_revision], ) - result = _run_bash( + result = run_bash( SMOKE_SCRIPT, "https://simulation.example.test", target_revision, old_revision, check=False, capture_output=True, - env=_smoke_test_env(tmp_path, state_file, revisions_file), + env=smoke_test_env(tmp_path, state_file, revisions_file), ) assert result.returncode == 1 @@ -225,18 +82,18 @@ def test_stable_smoke_rejects_an_unexpected_revision_without_retrying(tmp_path: def test_candidate_smoke_rejects_a_revision_mismatch_without_retrying(tmp_path: Path): old_revision = "policyengine-simulation-entry-00001-old" target_revision = "policyengine-simulation-entry-00002-new" - state_file, revisions_file = _write_fake_smoke_curl( + state_file, revisions_file = write_fake_smoke_curl( tmp_path, [old_revision, target_revision], ) - result = _run_bash( + result = run_bash( SMOKE_SCRIPT, "https://candidate.example.test", target_revision, check=False, capture_output=True, - env=_smoke_test_env(tmp_path, state_file, revisions_file), + env=smoke_test_env(tmp_path, state_file, revisions_file), ) assert result.returncode == 1 @@ -249,19 +106,19 @@ def test_stable_smoke_fails_when_the_previous_revision_does_not_converge( ): old_revision = "policyengine-simulation-entry-00001-old" target_revision = "policyengine-simulation-entry-00002-new" - state_file, revisions_file = _write_fake_smoke_curl( + state_file, revisions_file = write_fake_smoke_curl( tmp_path, [old_revision, old_revision, old_revision], ) - result = _run_bash( + result = run_bash( SMOKE_SCRIPT, "https://simulation.example.test", target_revision, old_revision, check=False, capture_output=True, - env=_smoke_test_env(tmp_path, state_file, revisions_file), + env=smoke_test_env(tmp_path, state_file, revisions_file), ) assert result.returncode == 1 @@ -410,1842 +267,3 @@ def test_deployment_uses_gcloud_workflow_without_terraform(): / "configure-cloud-run-simulation-entry-domains.sh" ).exists() _assert_stage12_deploy_targets(stage12_workflow) - - -def test_cloud_run_deployment_does_not_interpolate_github_values_in_shell(): - reusable_workflow = REUSABLE_DEPLOY_WORKFLOW.read_text(encoding="utf-8") - deployment_script = CLOUD_RUN_DEPLOY_SCRIPT.read_text(encoding="utf-8") - step_start = reusable_workflow.index( - " - name: Deploy Cloud Run entrypoint candidate" - ) - step_end = reusable_workflow.index( - "\n - name: Resolve exact candidate metadata", step_start - ) - deployment_step = reusable_workflow[step_start:step_end] - run_block = deployment_step[deployment_step.index("\n run:") :] - - assert "${{" not in run_block - assert ".github/scripts/deploy-cloud-run-simulation-entry.sh" in run_block - assert "--env-vars-file" in deployment_script - assert "--set-env-vars" not in deployment_script - assert "gcloud secrets versions list" in deployment_script - assert ":latest" not in deployment_script - - -def _cloud_run_deploy_environment(tmp_path: Path) -> dict[str, str]: - """Return a complete deployment environment backed by a fake gcloud.""" - fake_bin = tmp_path / "bin" - fake_bin.mkdir() - fake_gcloud = fake_bin / "gcloud" - fake_gcloud.write_text( - textwrap.dedent( - """\ - #!/usr/bin/env bash - set -euo pipefail - - if [[ "$1 $2 $3" == "secrets versions list" ]]; then - printf '%s\n' 1 3 2 - exit 0 - fi - - if [[ "$1 $2" == "run deploy" ]]; then - printf '%s\n' "$@" > "${GCLOUD_ARGUMENTS_CAPTURE}" - while (($#)); do - if [[ "$1" == "--env-vars-file" ]]; then - cp "$2" "${GCLOUD_ENV_FILE_CAPTURE}" - exit 0 - fi - shift - done - fi - - printf 'unexpected gcloud invocation\n' >&2 - exit 1 - """ - ), - encoding="utf-8", - ) - fake_gcloud.chmod(0o755) - environment = { - name: value - for name, value in os.environ.items() - if name != "SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS_VALUE" - } - return environment | { - "PATH": f"{fake_bin}:{os.environ['PATH']}", - "RUNNER_TEMP": str(tmp_path), - "GCLOUD_ARGUMENTS_CAPTURE": str(tmp_path / "gcloud-arguments.txt"), - "GCLOUD_ENV_FILE_CAPTURE": str(tmp_path / "runtime-environment.json"), - "PROJECT_ID": "policyengine-staging", - "REGION": "us-central1", - "IMAGE": "us-central1-docker.pkg.dev/project/repository/image:sha", - "TAG": "s-123", - "DEPLOY_STAGE12_V2": "true", - "APP_ENVIRONMENT": "staging", - "MODAL_ENVIRONMENT": "staging", - "ENTRYPOINT_SERVICE": "policyengine-simulation-entry-staging", - "ENTRYPOINT_RUNTIME_SERVICE_ACCOUNT": ( - "simulation-entry@policyengine-staging.iam.gserviceaccount.com" - ), - "ENTRYPOINT_MIN_INSTANCES": "0", - "ENTRYPOINT_MAX_INSTANCES": "10", - "SIMULATION_ENTRYPOINT_AUTH_ISSUER_VALUE": ( - "https://issuer.example/path?literal=$(not-a-command)" - ), - "SIMULATION_ENTRYPOINT_AUTH_AUDIENCE_VALUE": "simulation-api", - "OLD_GATEWAY_URL_VALUE": "https://legacy.example", - "OLD_GATEWAY_AUTH_ISSUER_VALUE": "https://issuer.example", - "OLD_GATEWAY_AUTH_AUDIENCE_VALUE": "legacy-api", - "OLD_GATEWAY_AUTH_CLIENT_ID_VALUE": "entrypoint-client", - "OLD_GATEWAY_AUTH_CLIENT_SECRET_SECRET_NAME": "old-client-secret", - "OBSERVABILITY_SERVICE_NAMESPACE": "policyengine.simulation-entry", - "OBSERVABILITY_TRACE_PROJECT_ID": "policyengine-observability", - "OTEL_EXPORTER_OTLP_ENDPOINT": "https://collector.example", - "POLICYENGINE_OTEL_GOOGLE_AUDIENCE": "https://collector.example", - "STAGE12_ENABLED_VALUE": "0", - "STAGE12_ARTIFACT_BUCKET_VALUE": "policyengine-stage12-staging", - "STAGE12_DATABASE_URL_SECRET_NAME": "stage12-staging-database-url", - "MODAL_TOKEN_ID_SECRET_NAME": "modal-token-id", - "MODAL_TOKEN_SECRET_SECRET_NAME": "modal-token-secret", - } - - -def test_cloud_run_deployment_escapes_environment_and_pins_secret_versions(tmp_path): - command_environment = _cloud_run_deploy_environment(tmp_path) - arguments_capture = Path(command_environment["GCLOUD_ARGUMENTS_CAPTURE"]) - env_capture = Path(command_environment["GCLOUD_ENV_FILE_CAPTURE"]) - - subprocess.run( - [CLOUD_RUN_DEPLOY_SCRIPT], - check=True, - env=command_environment, - ) - - runtime_environment = json.loads(env_capture.read_text(encoding="utf-8")) - assert runtime_environment["SIMULATION_ENTRYPOINT_AUTH_ISSUER"] == ( - "https://issuer.example/path?literal=$(not-a-command)" - ) - assert runtime_environment["STAGE12_ENABLED"] == "0" - assert runtime_environment["STAGE12_V2_MANIFEST_ENVIRONMENT"] == "staging" - assert runtime_environment["OBSERVABILITY_SERVICE_NAMESPACE"] == ( - "policyengine.simulation-entry" - ) - assert runtime_environment["OBSERVABILITY_TRACE_PROJECT_ID"] == ( - "policyengine-observability" - ) - assert runtime_environment["CLOUD_RUN_REGION"] == "us-central1" - assert runtime_environment["OTEL_EXPORTER_OTLP_ENDPOINT"] == ( - "https://collector.example" - ) - assert runtime_environment["POLICYENGINE_OTEL_GOOGLE_AUDIENCE"] == ( - "https://collector.example" - ) - assert "STAGE12_DATABASE_URL" not in runtime_environment - arguments = arguments_capture.read_text(encoding="utf-8").splitlines() - secrets_argument = arguments[arguments.index("--set-secrets") + 1] - assert secrets_argument == ( - "OLD_GATEWAY_AUTH_CLIENT_SECRET=old-client-secret:3," - "MODAL_TOKEN_ID=modal-token-id:3," - "MODAL_TOKEN_SECRET=modal-token-secret:3," - "STAGE12_DATABASE_URL=stage12-staging-database-url:3" - ) - assert ":latest" not in secrets_argument - - -@pytest.mark.parametrize( - ("configured", "expected"), - [(None, "0"), ("", "0"), ("0", "0"), ("1", "1")], - ids=["unset", "empty", "log-only", "enforced"], -) -def test_cloud_run_deployment_defaults_permission_enforcement_to_log_only( - tmp_path, - configured, - expected, -): - command_environment = _cloud_run_deploy_environment(tmp_path) - if configured is not None: - command_environment["SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS_VALUE"] = ( - configured - ) - - subprocess.run( - [CLOUD_RUN_DEPLOY_SCRIPT], - check=True, - env=command_environment, - ) - - runtime_environment = json.loads( - Path(command_environment["GCLOUD_ENV_FILE_CAPTURE"]).read_text(encoding="utf-8") - ) - assert runtime_environment["SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS"] == expected - - -@pytest.mark.parametrize("configured", ["true", "yes", "2", " 1"]) -def test_cloud_run_deployment_rejects_invalid_permission_enforcement( - tmp_path, - configured, -): - command_environment = _cloud_run_deploy_environment(tmp_path) - command_environment["SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS_VALUE"] = configured - - result = subprocess.run( - [CLOUD_RUN_DEPLOY_SCRIPT], - check=False, - env=command_environment, - capture_output=True, - text=True, - ) - - assert result.returncode != 0 - assert ( - "SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS_VALUE must be 0 or 1" - in result.stderr - ) - assert not Path(command_environment["GCLOUD_ENV_FILE_CAPTURE"]).exists() - - -def test_permission_enforcement_reaches_candidate_and_authenticated_tests(): - reusable_workflow = REUSABLE_DEPLOY_WORKFLOW.read_text(encoding="utf-8") - - assert ( - reusable_workflow.count( - "SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS_VALUE: " - "${{ vars.SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS }}" - ) - == 1 - ) - # Every deployed authentication run must know whether the target revision - # enforces permissions, so its enforcement check runs or skips correctly. - authenticated_runs = reusable_workflow.count( - "run: uv run --frozen pytest authenticated_tests/ -v" - ) - assert authenticated_runs == 3 - assert ( - reusable_workflow.count( - "SIMULATION_ENTRYPOINT_TEST_ENFORCE_PERMISSIONS: " - "${{ vars.SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS }}" - ) - == authenticated_runs - ) - - -def test_cloud_run_revision_tags_are_valid_on_the_first_workflow_run(): - deploy_workflow = DEPLOY_WORKFLOW.read_text(encoding="utf-8") - reusable_workflow = REUSABLE_DEPLOY_WORKFLOW.read_text(encoding="utf-8") - prefixes = re.findall( - r"^\s+entrypoint_revision_prefix:\s+(\S+)$", - deploy_workflow, - flags=re.MULTILINE, - ) - - assert ( - "tag=${{ inputs.entrypoint_revision_prefix }}${GITHUB_RUN_NUMBER}" - in reusable_workflow - ) - assert len(prefixes) == 2 - - first_run_tags = [f"{prefix}1" for prefix in prefixes] - for tag in first_run_tags: - assert 3 <= len(tag) <= 63 - assert re.fullmatch(r"[a-z][a-z0-9-]*[a-z0-9]", tag) - - -def test_revision_tags_fit_existing_cloud_run_service_names(): - deploy_workflow = DEPLOY_WORKFLOW.read_text(encoding="utf-8") - stage12_workflow = STAGE12_DEPLOY_WORKFLOW.read_text(encoding="utf-8") - deployments = { - "policyengine-simulation-entry-staging": "s-", - "policyengine-simulation-entry": "p-", - } - - for workflow in (deploy_workflow, stage12_workflow): - for service, prefix in deployments.items(): - assert f"entrypoint_service: {service}" in workflow - assert f"entrypoint_revision_prefix: {prefix}" in workflow - # Reserve seven digits for the monotonically increasing workflow - # run number while satisfying Cloud Run's service-plus-tag limit. - assert len(service) + len(f"{prefix}9999999") <= 46 - - -def test_entrypoint_container_uses_requested_target_platform(): - dockerfile = DOCKERFILE.read_text(encoding="utf-8") - - assert "$BUILDPLATFORM" not in dockerfile - - -def test_full_stack_promotion_order_is_explicit(): - deploy_workflow = DEPLOY_WORKFLOW.read_text(encoding="utf-8") - reusable_workflow = REUSABLE_DEPLOY_WORKFLOW.read_text(encoding="utf-8") - - for deployment_step, unit_test_step in ( - ("Deploy Cloud Run entrypoint candidate", "Run entrypoint unit tests"), - ("Deploy stable Modal gateway", "Run gateway unit tests"), - ("Deploy versioned Modal executor", "Run executor unit tests"), - ): - assert reusable_workflow.index(deployment_step) < reusable_workflow.index( - unit_test_step - ) - - routing_dependencies = ( - "needs: [prepare, deploy_entrypoint, deploy_gateway, deploy_executor]" - ) - assert routing_dependencies in reusable_workflow - gateway_job = reusable_workflow[ - reusable_workflow.index("\n deploy_gateway:") : reusable_workflow.index( - "\n deploy_executor:" - ) - ] - assert "needs: [prepare, deploy_executor]" in gateway_job - assert reusable_workflow.index("\n update_routing:") < reusable_workflow.index( - "\n integration:" - ) - assert ( - "stage12_v2_ready" - in reusable_workflow[ - reusable_workflow.index("\n integration:") : reusable_workflow.index( - "\n authenticated_test:" - ) - ] - ) - assert reusable_workflow.index("\n integration:") < reusable_workflow.index( - "\n authenticated_test:" - ) - assert reusable_workflow.index("\n authenticated_test:") < reusable_workflow.index( - "\n promote_entrypoint:" - ) - promote_needs = set( - _workflow_jobs(REUSABLE_DEPLOY_WORKFLOW)["promote_entrypoint"]["needs"] - ) - assert { - "prepare", - "deploy_entrypoint", - "deploy_executor", - "authenticated_test", - "sentinel_expected", - "update_routing", - } <= promote_needs - assert ( - 'cloud-run-simulation-entry-smoke.sh "${STABLE_URL}" "${TARGET_REVISION}" ' - '"${PREVIOUS_REVISION}"' in reusable_workflow - ) - assert ( - 'cloud-run-simulation-entry-smoke.sh "${CUSTOM_URL}" "${TARGET_REVISION}" ' - '"${PREVIOUS_REVISION}"' in reusable_workflow - ) - assert ( - "cloud-run-simulation-entry-smoke.sh " - '"${{ needs.deploy_entrypoint.outputs.candidate_url }}" ' - '"${{ needs.deploy_entrypoint.outputs.revision }}"' in reusable_workflow - ) - assert "failure() && steps.promote.outcome == 'success'" in reusable_workflow - assert "needs: beta" in deploy_workflow - assert "skip_beta" not in deploy_workflow - - -def test_stage12_only_deployment_runs_candidate_tests_and_requested_promotion(): - reusable_workflow = REUSABLE_DEPLOY_WORKFLOW.read_text(encoding="utf-8") - - authenticated_test = re.search( - r"^ authenticated_test:\n(?P.*?)(?=^ [a-z0-9_-]+:\n|\Z)", - reusable_workflow, - flags=re.MULTILINE | re.DOTALL, - ) - assert authenticated_test is not None - authenticated_test_body = authenticated_test.group("body") - assert "!cancelled()" in authenticated_test_body - assert "needs.deploy_entrypoint.result == 'success'" in authenticated_test_body - assert "needs.integration.result == 'success'" in authenticated_test_body - - promotion = re.search( - r"^ promote_entrypoint:\n(?P.*?)(?=^ [a-z0-9_-]+:\n|\Z)", - reusable_workflow, - flags=re.MULTILINE | re.DOTALL, - ) - assert promotion is not None - promotion_body = promotion.group("body") - assert "!cancelled()" in promotion_body - assert "inputs.promote_entrypoint" in promotion_body - assert "needs.deploy_entrypoint.result == 'success'" in promotion_body - assert "needs.authenticated_test.result == 'success'" in promotion_body - - -def test_reusable_deployment_rejects_unexpected_skipped_phases(): - reusable_workflow = REUSABLE_DEPLOY_WORKFLOW.read_text(encoding="utf-8") - completion = re.search( - r"^ deployment_ready:\n(?P.*?)(?=^ [a-z0-9_-]+:\n|\Z)", - reusable_workflow, - flags=re.MULTILINE | re.DOTALL, - ) - - assert completion is not None - completion_body = completion.group("body") - assert "if: ${{ !cancelled() }}" in completion_body - for dependency in ( - "deploy_entrypoint", - "integration", - "authenticated_test", - "promote_entrypoint", - ): - assert dependency in completion_body - assert "run: .github/scripts/simulation-deployment-ready.sh" in completion_body - for result in ( - "ENTRYPOINT_RESULT: ${{ needs.deploy_entrypoint.result }}", - "INTEGRATION_RESULT: ${{ needs.integration.result }}", - "AUTHENTICATED_TEST_RESULT: ${{ needs.authenticated_test.result }}", - "PROMOTION_RESULT: ${{ needs.promote_entrypoint.result }}", - ): - assert result in completion_body - readiness_check = DEPLOYMENT_READY_SCRIPT.read_text(encoding="utf-8") - for expected_result in ( - 'test "${ENTRYPOINT_RESULT}" = "success"', - 'test "${INTEGRATION_RESULT}" = "success"', - 'test "${AUTHENTICATED_TEST_RESULT}" = "success"', - 'test "${PROMOTION_RESULT}" = "success"', - 'test "${PROMOTION_RESULT}" = "skipped"', - ): - assert expected_result in readiness_check - - -def test_complete_integration_suite_is_configured_for_beta_only(): - deploy_workflow = DEPLOY_WORKFLOW.read_text(encoding="utf-8") - reusable_workflow = REUSABLE_DEPLOY_WORKFLOW.read_text(encoding="utf-8") - - beta_workflow = deploy_workflow[ - deploy_workflow.index(" beta:") : deploy_workflow.index(" prod:") - ] - prod_workflow = deploy_workflow[ - deploy_workflow.index(" prod:") : deploy_workflow.index(" summary:") - ] - - assert "run_full_integration: true" in beta_workflow - assert "run_full_integration: false" in prod_workflow - assert "run_full_integration:" in reusable_workflow - - -def test_stage12_only_deployment_skips_existing_stack_integration_suite(): - stage12_workflow = STAGE12_DEPLOY_WORKFLOW.read_text(encoding="utf-8") - - assert stage12_workflow.count("run_full_integration: false") == 2 - assert "run_full_integration: true" not in stage12_workflow - - -def test_main_deployment_automatically_deploys_stage12_in_both_environments(): - deploy_workflow = DEPLOY_WORKFLOW.read_text(encoding="utf-8") - reusable_workflow = REUSABLE_DEPLOY_WORKFLOW.read_text(encoding="utf-8") - - beta_workflow = deploy_workflow[ - deploy_workflow.index(" beta:") : deploy_workflow.index(" prod:") - ] - prod_workflow = deploy_workflow[ - deploy_workflow.index(" prod:") : deploy_workflow.index(" summary:") - ] - - assert "deploy_stage12_v2: true" in beta_workflow - assert "deploy_stage12_v2: true" in prod_workflow - assert deploy_workflow.count("deploy_stage12_v2: true") == 2 - assert "deploy_stage12_v2: false" not in deploy_workflow - - assert "src.modal.utils.set_stage12_dual_execution" not in reusable_workflow - assert "STAGE12_ENABLED_VALUE: ${{ vars.STAGE12_ENABLED }}" in reusable_workflow - - -def test_stage12_worker_deployment_can_request_a_google_oidc_token(): - reusable_workflow = REUSABLE_DEPLOY_WORKFLOW.read_text(encoding="utf-8") - match = re.search( - r"^ deploy_stage12_v2:\n(?P.*?)(?=^ [a-z0-9_-]+:\n|\Z)", - reusable_workflow, - flags=re.MULTILINE | re.DOTALL, - ) - - assert match is not None - section = match.group("body") - assert "permissions:\n contents: read\n id-token: write" in section - assert "google-github-actions/auth@v2" in section - - -def test_stage12_only_deployment_cannot_redeploy_existing_modal_resources(): - reusable_workflow = REUSABLE_DEPLOY_WORKFLOW.read_text(encoding="utf-8") - - for job_name in ("deploy_gateway", "deploy_executor", "update_routing"): - match = re.search( - rf"^ {job_name}:\n(?P.*?)(?=^ [a-z0-9_-]+:\n|\Z)", - reusable_workflow, - flags=re.MULTILINE | re.DOTALL, - ) - assert match is not None - section = match.group("body") - assert "if: ${{ inputs.deploy_existing_stack }}" in section - - stage12_section = reusable_workflow[ - reusable_workflow.index("\n deploy_stage12_v2:") : reusable_workflow.index( - "\n integration:" - ) - ] - assert "src.modal.utils.update_version_registry" not in stage12_section - finalizer_job = _job_section(reusable_workflow, "finalize_routing") - assert "inputs.deploy_existing_stack &&" in finalizer_job - sentinel_job = _job_section(reusable_workflow, "sentinel_expected") - assert "if: ${{ inputs.deploy_existing_stack }}" in sentinel_job - - -def _job_section(workflow: str, job_name: str) -> str: - match = re.search( - rf"^ {job_name}:\n(?P.*?)(?=^ [a-z0-9_-]+:\n|\Z)", - workflow, - flags=re.MULTILINE | re.DOTALL, - ) - assert match is not None, job_name - return match.group("body") - - -def _sentinel_check_step(job: dict) -> dict: - steps = [ - step - for step in job["steps"] - if "src.modal.utils.release_sentinel check" in step.get("run", "") - ] - assert len(steps) == 1 - return steps[0] - - -def test_deployed_stack_must_reproduce_the_numerical_sentinel(): - workflow = REUSABLE_DEPLOY_WORKFLOW.read_text(encoding="utf-8") - jobs = _workflow_jobs(REUSABLE_DEPLOY_WORKFLOW) - sentinel_job = _job_section(workflow, "sentinel_expected") - assert jobs["sentinel_expected"]["needs"] == "prepare" - assert "release_sentinel expected" in sentinel_job - assert "secrets.PE_UK_PRIVATE_HF_READ_TOKEN" in sentinel_job - assert "secrets.GCP_CREDENTIALS_JSON" in sentinel_job - assert ".uk.dataset_sha256 == $d" in sentinel_job - - integration = jobs["integration"] - assert "needs.sentinel_expected.result == 'success'" in integration["if"] - assert "needs.sentinel_expected.result == 'skipped'" in integration["if"] - check = _sentinel_check_step(integration) - # Both checks run on every existing-stack deployment, after the identity - # smoke, against the candidate and then the stable URL. - assert check["if"] == "${{ inputs.deploy_existing_stack }}" - assert "needs.deploy_entrypoint.outputs.candidate_url" in check["run"] - steps = integration["steps"] - smoke = next( - index - for index, step in enumerate(steps) - if "cloud-run-simulation-entry-smoke.sh" in step.get("run", "") - ) - assert smoke < steps.index(check) - - promote = jobs["promote_entrypoint"] - stable = _sentinel_check_step(promote) - assert stable["if"] == "${{ inputs.deploy_existing_stack }}" - assert '--base-url "${STABLE_URL}"' in stable["run"] - names = [step.get("name", "") for step in promote["steps"]] - # A failing stable sentinel triggers the existing traffic restore step. - assert promote["steps"].index(stable) < next( - index - for index, name in enumerate(names) - if name.startswith("Restore previous revision") - ) - - -# A small evaluator for the GitHub Actions expression subset these workflows -# use (&&, ||, !, ==, !=, parentheses, status functions, property access and -# string literals), so conditions are tested by what they decide, not by -# their text. It follows GitHub's rules where they matter here: a job ``if`` -# with no status function is implicitly ``success() && (...)``; ``==`` -# compares strings case-insensitively and converts mismatched types to -# numbers (true is 1, null is 0, a non-numeric string is NaN, so -# ``true == 'true'`` is false). -_TOKEN = re.compile(r"\s*(\$\{\{|\}\}|&&|\|\||==|!=|!|\(|\)|'[^']*'|[A-Za-z_][\w.-]*)") -_STATUS_FUNCTIONS = ("always", "success", "failure", "cancelled") - - -def _gh_number(value): - if value is None: - return 0.0 - if isinstance(value, bool): - return 1.0 if value else 0.0 - if isinstance(value, (int, float)): - return float(value) - text = str(value).strip() - if text == "": - return 0.0 - try: - return float(text) - except ValueError: - return float("nan") - - -def _gh_equal(left, right) -> bool: - if isinstance(left, str) and isinstance(right, str): - return left.lower() == right.lower() - if type(left) is type(right): - return left == right - return _gh_number(left) == _gh_number(right) - - -def _gh_tokens(expression: str) -> list[str]: - """Split an expression into tokens, dropping its ``${{`` and ``}}``.""" - tokens = [] - position = 0 - text = expression.strip() - while position < len(text): - match = _TOKEN.match(text, position) - assert match, text[position:] - tokens.append(match.group(1)) - position = match.end() - return [token for token in tokens if token not in ("${{", "}}")] - - -def _gh_calls_a_status_function(tokens: list[str]) -> bool: - return any( - token in _STATUS_FUNCTIONS and tokens[index + 1 : index + 2] == ["("] - for index, token in enumerate(tokens) - ) - - -def _gh_truthy(value) -> bool: - return value not in (None, False, "", 0) - - -class _GitHubExpression: - """A recursive-descent evaluation of one tokenized expression in a context. - - Each ``_parse_*`` method reads from a token index and returns the value - it read and the index after it, from the loosest operator (``||``) to the - tightest (``!``, parentheses and operands). - """ - - def __init__(self, tokens: list[str], context: dict) -> None: - self.tokens = tokens - self.context = context - self.results = [ - job.get("result") for job in (context.get("needs") or {}).values() - ] - - def evaluate(self): - value, index = self._parse_or(0) - assert index == len(self.tokens), self.tokens[index:] - return value - - def _next_is(self, index: int, operators: tuple[str, ...]) -> bool: - return index < len(self.tokens) and self.tokens[index] in operators - - def _status(self, name: str) -> bool: - cancelled = bool(self.context.get("cancelled")) - if name == "always": - return True - if name == "cancelled": - return cancelled - if name == "failure": - return "failure" in self.results - return not cancelled and all(result == "success" for result in self.results) - - def _lookup(self, path: str): - if path in ("true", "false"): - return path == "true" - value = self.context - for part in path.split("."): - value = value.get(part) if isinstance(value, dict) else None - return value - - def _parse_or(self, index: int): - left, index = self._parse_and(index) - while self._next_is(index, ("||",)): - right, index = self._parse_and(index + 1) - left = left if _gh_truthy(left) else right - return left, index - - def _parse_and(self, index: int): - left, index = self._parse_compare(index) - while self._next_is(index, ("&&",)): - right, index = self._parse_compare(index + 1) - left = right if _gh_truthy(left) else left - return left, index - - def _parse_compare(self, index: int): - left, index = self._parse_unary(index) - while self._next_is(index, ("==", "!=")): - operator = self.tokens[index] - right, index = self._parse_unary(index + 1) - equal = _gh_equal(left, right) - left = equal if operator == "==" else not equal - return left, index - - def _parse_unary(self, index: int): - symbol = self.tokens[index] - if symbol == "!": - value, index = self._parse_unary(index + 1) - return not _gh_truthy(value), index - if symbol == "(": - value, index = self._parse_or(index + 1) - assert self.tokens[index] == ")" - return value, index + 1 - if symbol.startswith("'"): - return symbol[1:-1], index + 1 - if self.tokens[index + 1 : index + 3] == ["(", ")"]: - assert symbol in _STATUS_FUNCTIONS, symbol - return self._status(symbol), index + 3 - return self._lookup(symbol), index + 1 - - -def _gh_eval(expression: str, context: dict, *, job_if: bool = False): - tokens = _gh_tokens(expression) - if job_if and not _gh_calls_a_status_function(tokens): - tokens = ["success", "(", ")", "&&", "(", *tokens, ")"] - return _GitHubExpression(tokens, context).evaluate() - - -def test_the_expression_evaluator_follows_github_semantics(): - context = {"inputs": {"flag": True}, "needs": {"a": {"result": "failure"}}} - assert _gh_eval("${{ inputs.flag == 'true' }}", context) is False - assert _gh_eval("${{ inputs.flag == true }}", context) is True - assert _gh_eval("${{ 'SUCCESS' == 'success' }}", context) is True - assert _gh_eval("${{ inputs.missing == '' }}", context) is True - # No status function: implicitly success() first, so a failed need skips it. - assert not _gh_eval("${{ inputs.flag }}", context, job_if=True) - assert _gh_eval("${{ always() && inputs.flag }}", context, job_if=True) - assert not _gh_eval("${{ !cancelled() }}", {**context, "cancelled": True}) - - -def test_the_expression_evaluator_returns_operands_and_job_statuses(): - context = { - "inputs": {"name": "beta", "empty": "", "zero": 0, "flag": False}, - "needs": {"a": {"result": "success"}, "b": {"result": "skipped"}}, - "steps": "not-a-mapping", - } - # || and && return one of their operands, as GitHub does, not a boolean. - assert _gh_eval("${{ inputs.empty || 'fallback' }}", context) == "fallback" - assert _gh_eval("${{ inputs.name || 'fallback' }}", context) == "beta" - assert _gh_eval("${{ inputs.name && inputs.zero }}", context) == 0 - assert _gh_eval("${{ inputs.flag && inputs.name }}", context) is False - assert _gh_eval("${{ inputs.name != 'BETA' }}", context) is False - assert _gh_eval("${{ !(inputs.empty || inputs.zero) }}", context) is True - assert _gh_eval("${{ steps.x.outputs }}", context) is None - assert _gh_eval("${{ inputs.missing.deeper }}", context) is None - assert _gh_eval("false", context) is False - # success() needs every needed job to succeed; failure() needs one failure. - assert _gh_eval("${{ success() }}", context) is False - assert _gh_eval("${{ failure() }}", context) is False - assert _gh_eval("${{ success() }}", {}) is True - assert _gh_eval("${{ success() }}", {"cancelled": True}) is False - assert _gh_eval("${{ cancelled() }}", {"cancelled": True}) is True - # Any status function in a job if replaces the implicit success(). - assert _gh_eval("${{ inputs.name == 'beta' }}", context, job_if=True) is False - assert ( - _gh_eval("${{ inputs.name == 'beta' || failure() }}", context, job_if=True) - is True - ) - - -@pytest.mark.parametrize( - "expression", - ["${{ unknown() }}", "${{ inputs.name inputs.name }}", "${{ inputs.name @ }}"], - ids=["unknown-function", "trailing-token", "unknown-character"], -) -def test_the_expression_evaluator_refuses_what_it_cannot_read(expression): - with pytest.raises(AssertionError): - _gh_eval(expression, {"inputs": {"name": "beta"}}) - - -def _routing_context(**overrides): - context = { - "inputs": {"deploy_existing_stack": True, "promote_entrypoint": True}, - "github": {"run_id": "744001", "run_attempt": "1"}, - "needs": { - "update_routing": { - "result": "success", - "outputs": { - "before_sha256": "a" * 64, - "after_sha256": "b" * 64, - "advances_latest": "true", - }, - }, - "integration": {"result": "success"}, - "authenticated_test": {"result": "success"}, - "promote_entrypoint": {"result": "success"}, - }, - } - for path, value in overrides.items(): - target = context - *parents, last = path.split("__") - for part in parents: - target = target[part] - target[last] = value - return context - - -@pytest.mark.parametrize( - ("overrides", "finalizes", "gates_passed"), - [ - ({}, True, "true"), - ({"needs__integration__result": "failure"}, True, "false"), - ({"needs__authenticated_test__result": "cancelled"}, True, "false"), - ({"needs__promote_entrypoint__result": "failure"}, True, "false"), - # Publication then a failure inside update_routing itself (#744 H2). - ({"needs__update_routing__result": "failure"}, True, "false"), - ( - { - "inputs__promote_entrypoint": False, - "needs__promote_entrypoint__result": "skipped", - }, - True, - "true", - ), - # Missing job outputs cannot prove nothing was published: runner loss - # can prevent reporting them even after upload and apply succeeded. - ( - { - "needs__update_routing__result": "failure", - "needs__update_routing__outputs": {}, - }, - True, - "false", - ), - ( - { - "needs__update_routing__result": "failure", - "needs__update_routing__outputs": {}, - "needs__integration__result": "skipped", - "needs__authenticated_test__result": "skipped", - "needs__promote_entrypoint__result": "skipped", - }, - True, - "false", - ), - # The restore does not depend on the plan's digests: even a plan that - # reported equal ones must be restored, because the restore is what - # resolves the release's publication record, which would otherwise - # refuse every later deployment. - ( - { - "needs__integration__result": "failure", - "needs__update_routing__outputs__after_sha256": "a" * 64, - }, - True, - "false", - ), - ( - { - "inputs__deploy_existing_stack": False, - "needs__integration__result": "failure", - }, - False, - "false", - ), - ( - {"needs__update_routing__result": "skipped"}, - False, - "false", - ), - # Stage12 alone never publishes, even when all its gates pass. - ( - { - "inputs__deploy_existing_stack": False, - "needs__update_routing__result": "skipped", - }, - False, - "false", - ), - # Re-run failed jobs carries a successful attempt-1 publication and - # its outputs. Finalization must use the run, never attempt-2 artifacts. - ( - {"github__run_attempt": "2"}, - True, - "true", - ), - ( - { - "github__run_attempt": "2", - "needs__authenticated_test__result": "failure", - }, - True, - "false", - ), - # A cancelled run after publication must still restore routing. - ( - { - "cancelled": True, - "needs__integration__result": "cancelled", - "needs__authenticated_test__result": "skipped", - "needs__promote_entrypoint__result": "skipped", - }, - True, - "false", - ), - ], - ids=[ - "all-pass", - "integration", - "authenticated", - "promotion", - "routing-job", - "no-promotion", - "missing-outputs", - "runner-lost-all-outputs", - "equal-digests", - "stack-disabled", - "routing-skipped", - "stage12-only", - "attempt-2-success", - "attempt-2-failed-gate", - "cancelled", - ], -) -def test_finalizer_always_resolves_a_publication_with_the_release_gate_outcome( - overrides, finalizes, gates_passed -): - finalizer = _workflow_jobs(REUSABLE_DEPLOY_WORKFLOW)["finalize_routing"] - step = next( - step - for step in finalizer["steps"] - if "--finalize-publication" in step.get("run", "") - ) - context = _routing_context(**overrides) - assert bool(_gh_eval(finalizer["if"], context, job_if=True)) is finalizes - assert str(_gh_eval(step["env"]["GATES_PASSED"], context)).lower() == (gates_passed) - - -@pytest.mark.parametrize("job", ["integration", "promote_entrypoint"]) -def test_a_failing_sentinel_fails_its_step(tmp_path, job): - """#744 review: `|| true` on the sentinel command must not pass a failure.""" - step = _sentinel_check_step(_workflow_jobs(REUSABLE_DEPLOY_WORKFLOW)[job]) - bin_dir = tmp_path / "bin" - bin_dir.mkdir() - (bin_dir / "uv").write_text( - "#!/usr/bin/env bash\n" - 'if [[ "$*" == *"release_sentinel check"* ]]; then exit 9; fi\n' - "exit 0\n" - ) - (bin_dir / "uv").chmod(0o755) - script = step["run"].replace( - "${{ needs.deploy_entrypoint.outputs.candidate_url }}", "https://candidate" - ) - result = _run_bash( - # GitHub runs `run` blocks with bash -e -o pipefail. - "--noprofile", - "--norc", - "-eo", - "pipefail", - "-c", - script, - check=False, - capture_output=True, - env={ - "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}", - "RUNNER_TEMP": str(tmp_path), - "STABLE_URL": "https://stable", - }, - ) - assert result.returncode == 9 - - -def test_stage12_only_promotion_runs_whatever_the_skipped_stack_jobs(): - """#743 review: the explicit status function must stay active.""" - promote = _workflow_jobs(REUSABLE_DEPLOY_WORKFLOW)["promote_entrypoint"] - context = { - "inputs": {"promote_entrypoint": True, "deploy_existing_stack": False}, - "needs": { - "prepare": {"result": "success"}, - "deploy_entrypoint": {"result": "success"}, - "deploy_executor": {"result": "skipped"}, - "authenticated_test": {"result": "success"}, - "sentinel_expected": {"result": "skipped"}, - "update_routing": {"result": "skipped"}, - }, - } - assert _gh_eval(promote["if"], context, job_if=True) - context["needs"]["authenticated_test"]["result"] = "failure" - assert not _gh_eval(promote["if"], context, job_if=True) - context["needs"]["authenticated_test"]["result"] = "success" - context["inputs"]["promote_entrypoint"] = False - assert not _gh_eval(promote["if"], context, job_if=True) - - -def test_smoke_rejects_a_latest_wrapper_that_lags_alone(tmp_path: Path): - """#743 review: the wrapper's latest is checked, not only the model's.""" - import copy - - versions = copy.deepcopy(VERSIONS_BODY) - versions["policyengine"]["latest"] = "6.2.6" - - result = _run_identity_smoke(tmp_path, versions, **IDENTITY) - - assert result.returncode == 1 - assert "latest routes to policyengine 6.2.6 and policyengine-uk 2.126.8" in ( - result.stderr - ) - - -def _run_routing_step_with_a_failing_command(tmp_path, step, failed_command, **env): - """Run the real body with Actions' shell and a failing CLI seam.""" - bin_dir = tmp_path / "bin" - bin_dir.mkdir() - (bin_dir / "uv").write_text( - "#!/usr/bin/env bash\n" - 'printf "%s\\n" "$*" >> "$UV_CALLS"\n' - # Valid output ensures a masked plan/identity failure cannot be - # caught accidentally by a later grep rather than the failed CLI. - "printf 'before_sha256=before\\nafter_sha256=after\\nadvances_latest=true\\n'\n" - "printf 'policyengine_version=6.3.0\\napp_name=simulation-6.3.0\\n'\n" - "printf 'uk_model_version=2.126.8\\nuk_dataset_sha256=abc\\n'\n" - 'if [[ " $* " == *" ${FAILED_COMMAND} "* ]]; then exit 9; fi\n' - "exit 0\n" - ) - (bin_dir / "uv").chmod(0o755) - (bin_dir / "gh").write_text("#!/usr/bin/env bash\nprintf '1\\n'\n") - (bin_dir / "gh").chmod(0o755) - runner_temp = tmp_path / "runner-temp" - plan = runner_temp / "routing-plans" / "routing-plan-prod-1" / "plan.json" - plan.parent.mkdir(parents=True) - plan.write_text("{}") - result = _run_bash( - "--noprofile", - "--norc", - "-eo", - "pipefail", - "-c", - step["run"], - check=False, - capture_output=True, - env={ - "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}", - "RUNNER_TEMP": str(runner_temp), - "GITHUB_OUTPUT": str(tmp_path / "github-output"), - "GITHUB_RUN_ID": "744001", - "GITHUB_RUN_ATTEMPT": "2", - "GITHUB_REPOSITORY": "PolicyEngine/policyengine-sim-api", - "UV_CALLS": str(tmp_path / "uv-calls"), - "FAILED_COMMAND": failed_command, - "MODAL_ENVIRONMENT": "main", - "PUBLICATION_RUN": "744001", - "RELEASE_ENVIRONMENT": "prod", - "GATES_PASSED": "true", - "APP_NAME": "simulation-6.3.0", - "POLICYENGINE_VERSION": "6.3.0", - "POLICYENGINE_US_VERSION": "1.0.0", - "POLICYENGINE_UK_VERSION": "2.126.8", - "FORCE_LATEST": "false", - "TARGET": "prod", - "DEPLOY_RUN_ID": "", - "EXPECTED_LATEST": "", - "RESOLVE_PENDING_PUBLICATION": "false", - **env, - }, - ) - assert result.returncode == 9, result.stderr - assert failed_command in (tmp_path / "uv-calls").read_text() - - -@pytest.mark.parametrize( - ("workflow", "job_name", "command", "env"), - [ - (REUSABLE_DEPLOY_WORKFLOW, "update_routing", "sync", {}), - (REUSABLE_DEPLOY_WORKFLOW, "finalize_routing", "sync", {}), - (ROLLBACK_WORKFLOW, "rollback", "sync", {}), - (REUSABLE_DEPLOY_WORKFLOW, "update_routing", "--plan-out", {}), - (REUSABLE_DEPLOY_WORKFLOW, "update_routing", "--record-plan", {}), - (REUSABLE_DEPLOY_WORKFLOW, "update_routing", "--apply-plan", {}), - (REUSABLE_DEPLOY_WORKFLOW, "finalize_routing", "--finalize-publication", {}), - ( - ROLLBACK_WORKFLOW, - "rollback", - "--restore-plan", - {"DEPLOY_RUN_ID": "744001"}, - ), - (ROLLBACK_WORKFLOW, "rollback", "--restore-latest", {}), - (ROLLBACK_WORKFLOW, "rollback", "--print-identity", {}), - ( - ROLLBACK_WORKFLOW, - "rollback", - "--print-pending-plan", - {"RESOLVE_PENDING_PUBLICATION": "true"}, - ), - ( - ROLLBACK_WORKFLOW, - "rollback", - "--finalize-publication", - {"RESOLVE_PENDING_PUBLICATION": "true"}, - ), - ], - ids=[ - "publication-setup", - "finalizer-setup", - "manual-rollback-setup", - "plan", - "record", - "apply", - "finalize", - "manual-restore-plans", - "manual-restore-latest", - "manual-read-identity", - "operator-inspect-pending", - "operator-resolve-pending", - ], -) -def test_a_failing_routing_command_fails_its_step_and_job( - tmp_path, workflow, job_name, command, env -): - job = _workflow_jobs(workflow)[job_name] - assert "continue-on-error" not in job - assert job["defaults"]["run"]["shell"] == "bash" - step = next(step for step in job["steps"] if command in step.get("run", "")) - assert "continue-on-error" not in step - assert step.get("shell", "bash") == "bash" - _run_routing_step_with_a_failing_command(tmp_path, step, command, **env) - - -@pytest.mark.parametrize( - ("workflow", "job_name"), - [ - (REUSABLE_DEPLOY_WORKFLOW, "update_routing"), - (REUSABLE_DEPLOY_WORKFLOW, "finalize_routing"), - (REUSABLE_DEPLOY_WORKFLOW, "deployment_ready"), - (ROLLBACK_WORKFLOW, "rollback"), - ], -) -def test_routing_jobs_never_allow_command_failures(workflow, job_name): - job = _workflow_jobs(workflow)[job_name] - assert "continue-on-error" not in job - assert job.get("defaults", {}).get("run", {}).get("shell") == "bash" - for step in job["steps"]: - assert "continue-on-error" not in step - assert step.get("shell", "bash") == "bash" - body = step.get("run", "") - assert not re.search(r"\|\|\s*true\b", body) - assert not re.search(r"\bset\s+\+e\b", body) - - -def test_each_publication_is_named_by_its_run_attempt_and_environment(): - routing = _workflow_jobs(REUSABLE_DEPLOY_WORKFLOW)["update_routing"] - plan = next( - step - for step in routing["steps"] - if step.get("name") == "Plan the routing change" - ) - assert ( - '--publication-id "${GITHUB_RUN_ID}-' - '${GITHUB_RUN_ATTEMPT}-${RELEASE_ENVIRONMENT}"' in plan["run"] - ) - assert plan["env"]["RELEASE_ENVIRONMENT"] == "${{ inputs.release_environment }}" - - -def test_publication_keeps_the_plan_but_finalization_reads_the_journal(): - jobs = _workflow_jobs(REUSABLE_DEPLOY_WORKFLOW) - routing = jobs["update_routing"] - names = [step.get("name") for step in routing["steps"]] - plan = names.index("Plan the routing change") - journal = names.index("Record the unfinished routing publication") - keep = names.index("Keep the plan for a guarded restore") - publish = names.index("Publish the planned routing") - assert "Record the plan artifact" not in names - # The plan is kept before anything is published. - assert plan < journal < keep < publish - assert "--record-plan" in routing["steps"][journal]["run"] - keep_step = routing["steps"][keep] - assert keep_step["uses"].startswith("actions/upload-artifact@") - assert keep_step["with"]["if-no-files-found"] == "error" - assert keep_step["with"]["name"] == ( - "routing-plan-${{ inputs.release_environment }}-${{ github.run_attempt }}" - ) - assert "--plan-out" in routing["steps"][plan]["run"] - assert "--apply-plan" in routing["steps"][publish]["run"] - assert routing["outputs"] == { - "before_sha256": "${{ steps.plan.outputs.before_sha256 }}", - "after_sha256": "${{ steps.plan.outputs.after_sha256 }}", - "advances_latest": "${{ steps.plan.outputs.advances_latest }}", - } - - assert "revert_routing" not in jobs - finalizer = jobs["finalize_routing"] - finalize = next( - step - for step in finalizer["steps"] - if "--finalize-publication" in step.get("run", "") - ) - assert not any( - step.get("uses", "").startswith("actions/download-artifact@") - for step in finalizer["steps"] - ) - assert '--publication-run "${PUBLICATION_RUN}"' in finalize["run"] - assert '--release-environment "${RELEASE_ENVIRONMENT}"' in finalize["run"] - assert '--gates-passed "${GATES_PASSED}"' in finalize["run"] - assert '--github-output "$GITHUB_OUTPUT"' in finalize["run"] - assert finalize["env"]["PUBLICATION_RUN"] == "${{ github.run_id }}" - assert finalize["env"]["RELEASE_ENVIRONMENT"] == ( - "${{ inputs.release_environment }}" - ) - assert "github.run_attempt" not in json.dumps(finalizer) - assert finalizer["outputs"]["outcome"] == ("${{ steps.finalize.outputs.outcome }}") - assert finalize["id"] == "finalize" - - cleanup = _job_section( - REUSABLE_DEPLOY_WORKFLOW.read_text(encoding="utf-8"), - "cleanup_entrypoint_revisions", - ) - assert ( - "SIMULATION_ENTRYPOINT_ROLLBACK_REVISION: " - "${{ needs.deploy_entrypoint.outputs.previous_revision }}" - ) in cleanup - - -def test_deployment_ready_requires_the_journal_finalizer_to_complete(): - jobs = _workflow_jobs(REUSABLE_DEPLOY_WORKFLOW) - ready = jobs["deployment_ready"] - assert { - "update_routing", - "integration", - "authenticated_test", - "promote_entrypoint", - "finalize_routing", - } <= set(ready["needs"]) - finalizer = jobs["finalize_routing"] - assert { - "update_routing", - "integration", - "authenticated_test", - "promote_entrypoint", - } <= set(finalizer["needs"]) - gate = next( - step for step in ready["steps"] if step.get("name") == "Check deployment result" - ) - assert "continue-on-error" not in gate - assert gate["env"]["EXISTING_STACK_REQUESTED"] == ( - "${{ inputs.deploy_existing_stack }}" - ) - assert gate["env"]["ROUTING_RESULT"] == "${{ needs.update_routing.result }}" - assert gate["env"]["FINALIZE_RESULT"] == "${{ needs.finalize_routing.result }}" - assert gate["env"]["FINALIZE_OUTCOME"] == ( - "${{ needs.finalize_routing.outputs.outcome }}" - ) - assert not any( - "--complete-plan" in step.get("run", "") - or step.get("uses", "").startswith("actions/download-artifact@") - for step in ready["steps"] - ) - - -@pytest.mark.parametrize( - "failed_gate", - [ - "ENTRYPOINT_RESULT", - "INTEGRATION_RESULT", - "AUTHENTICATED_TEST_RESULT", - "PROMOTION_RESULT", - "ROUTING_RESULT", - "FINALIZE_RESULT", - "FINALIZE_OUTCOME", - ], -) -def test_failed_release_gate_cannot_report_a_successful_release(tmp_path, failed_gate): - ready = _workflow_jobs(REUSABLE_DEPLOY_WORKFLOW)["deployment_ready"] - gate = next( - step for step in ready["steps"] if step.get("name") == "Check deployment result" - ) - marker = tmp_path / "completed" - env = { - "PATH": os.environ["PATH"], - "PROMOTION_REQUESTED": "true", - "ENTRYPOINT_RESULT": "success", - "INTEGRATION_RESULT": "success", - "AUTHENTICATED_TEST_RESULT": "success", - "PROMOTION_RESULT": "success", - "EXISTING_STACK_REQUESTED": "true", - "ROUTING_RESULT": "success", - "FINALIZE_RESULT": "success", - "FINALIZE_OUTCOME": "completed", - } - env[failed_gate] = "failure" - # A later step cannot run after this gate under Actions' bash -e behavior. - result = _run_bash( - "-eo", - "pipefail", - "-c", - gate["run"] + '\ntouch "$1"', - "gate", - str(marker), - check=False, - capture_output=True, - env=env, - cwd=REPOSITORY_ROOT, - ) - assert result.returncode != 0 - assert not marker.exists() - - -@pytest.mark.parametrize( - ("existing", "result", "outcome", "passes"), - [ - ("true", "success", "completed", True), - ("true", "success", "restored", False), - ("true", "success", "no-pending", False), - ("true", "success", "foreign-publication", False), - ("true", "failure", "completed", False), - ("true", "cancelled", "", False), - ("true", "skipped", "", False), - ("false", "skipped", "", True), - ], -) -def test_release_success_requires_completed_finalization_for_published_routing( - existing, result, outcome, passes -): - gate = next( - step - for step in _workflow_jobs(REUSABLE_DEPLOY_WORKFLOW)["deployment_ready"][ - "steps" - ] - if step.get("name") == "Check deployment result" - ) - completed = _run_bash( - "-eo", - "pipefail", - "-c", - gate["run"], - check=False, - capture_output=True, - cwd=REPOSITORY_ROOT, - env={ - "PATH": os.environ["PATH"], - "PROMOTION_REQUESTED": "false", - "ENTRYPOINT_RESULT": "success", - "INTEGRATION_RESULT": "success", - "AUTHENTICATED_TEST_RESULT": "success", - "PROMOTION_RESULT": "skipped", - "EXISTING_STACK_REQUESTED": existing, - "ROUTING_RESULT": "success" if existing == "true" else "skipped", - "FINALIZE_RESULT": result, - "FINALIZE_OUTCOME": outcome, - }, - ) - assert (completed.returncode == 0) is passes, completed.stderr - - -@pytest.mark.parametrize( - ("existing", "advances", "expected"), - [(True, "true", "true"), (True, "false", ""), (False, "true", ""), (True, "", "")], -) -def test_latest_is_expected_only_when_the_release_advances_it( - existing, advances, expected -): - jobs = _workflow_jobs(REUSABLE_DEPLOY_WORKFLOW) - smoke_step = next( - step - for step in jobs["integration"]["steps"] - if "cloud-run-simulation-entry-smoke.sh" in step.get("run", "") - ) - context = _routing_context( - inputs__deploy_existing_stack=existing, - needs__update_routing__outputs__advances_latest=advances, - ) - for environment in (smoke_step["env"], jobs["promote_entrypoint"]["env"]): - assert _gh_eval(environment["EXPECTED_LATEST"], context) == expected - - -def test_traffic_changes_are_exact_revision_validated_and_reversible(tmp_path): - script = TRAFFIC_SCRIPT.read_text(encoding="utf-8") - - assert "run revisions describe" in script - assert 'select(.type == "Ready" and .status == "True")' in script - assert '["serving.knative.dev/service"]' in script - assert 'run services update-traffic "${service}"' in script - assert '--to-revisions "${target_revision}=100"' in script - assert "expected_current_revision" in script - assert '--to-revisions "LATEST=100"' not in script - - state_file = tmp_path / "active-revision" - state_file.write_text("policyengine-simulation-entry-00001-old", encoding="utf-8") - fake_gcloud = tmp_path / "gcloud" - fake_gcloud.write_text( - textwrap.dedent("""\ - #!/usr/bin/env bash - set -euo pipefail - case "$1 $2 $3" in - "run services describe") - active="$(cat "${FAKE_GCLOUD_STATE}")" - printf '{"status":{"traffic":[{"revisionName":"%s","percent":100}]}}\\n' "${active}" - ;; - "run revisions describe") - revision="$4" - printf '{"metadata":{"labels":{"serving.knative.dev/service":"policyengine-simulation-entry"}},"status":{"conditions":[{"type":"Ready","status":"True"}]},"revision":"%s"}\\n' "${revision}" - ;; - "run services update-traffic") - target="" - while [ "$#" -gt 0 ]; do - if [ "$1" = "--to-revisions" ]; then - target="${2%=100}" - break - fi - shift - done - test -n "${target}" - printf '%s' "${target}" > "${FAKE_GCLOUD_STATE}" - ;; - *) - printf 'Unexpected fake gcloud command: %s\\n' "$*" >&2 - exit 2 - ;; - esac - """), # noqa: E501 # the fake gcloud prints each JSON response on one line, as gcloud does - encoding="utf-8", - ) - fake_gcloud.chmod(0o755) - - base_env = { - **os.environ, - "GCLOUD_BIN": str(fake_gcloud), - "FAKE_GCLOUD_STATE": str(state_file), - "SIMULATION_ENTRYPOINT_GCP_PROJECT_ID": "simulation-entry-test", - "SIMULATION_ENTRYPOINT_SERVICE": "policyengine-simulation-entry", - } - - _run_bash( - TRAFFIC_SCRIPT, - check=True, - capture_output=True, - env={ - **base_env, - "SIMULATION_ENTRYPOINT_TARGET_REVISION": ( - "policyengine-simulation-entry-00002-new" - ), - "SIMULATION_ENTRYPOINT_EXPECTED_CURRENT_REVISION": ( - "policyengine-simulation-entry-00001-old" - ), - }, - ) - assert ( - state_file.read_text(encoding="utf-8") - == "policyengine-simulation-entry-00002-new" - ) - - _run_bash( - TRAFFIC_SCRIPT, - check=True, - capture_output=True, - env={ - **base_env, - "SIMULATION_ENTRYPOINT_TARGET_REVISION": ( - "policyengine-simulation-entry-00001-old" - ), - "SIMULATION_ENTRYPOINT_EXPECTED_CURRENT_REVISION": ( - "policyengine-simulation-entry-00002-new" - ), - }, - ) - assert ( - state_file.read_text(encoding="utf-8") - == "policyengine-simulation-entry-00001-old" - ) - - -def test_traffic_change_refuses_an_intervening_promotion(tmp_path): - state_file = tmp_path / "active-revision" - state_file.write_text( - "policyengine-simulation-entry-00003-intervening", - encoding="utf-8", - ) - fake_gcloud = tmp_path / "gcloud" - fake_gcloud.write_text( - textwrap.dedent("""\ - #!/usr/bin/env bash - set -euo pipefail - active="$(cat "${FAKE_GCLOUD_STATE}")" - printf '{"status":{"traffic":[{"revisionName":"%s","percent":100}]}}\\n' "${active}" - """), # noqa: E501 # the fake gcloud prints its JSON response on one line, as gcloud does - encoding="utf-8", - ) - fake_gcloud.chmod(0o755) - - result = _run_bash( - TRAFFIC_SCRIPT, - check=False, - capture_output=True, - env={ - **os.environ, - "GCLOUD_BIN": str(fake_gcloud), - "FAKE_GCLOUD_STATE": str(state_file), - "SIMULATION_ENTRYPOINT_GCP_PROJECT_ID": "simulation-entry-test", - "SIMULATION_ENTRYPOINT_SERVICE": "policyengine-simulation-entry", - "SIMULATION_ENTRYPOINT_TARGET_REVISION": ( - "policyengine-simulation-entry-00002-new" - ), - "SIMULATION_ENTRYPOINT_EXPECTED_CURRENT_REVISION": ( - "policyengine-simulation-entry-00001-old" - ), - }, - ) - - assert result.returncode == 2 - assert "Stable traffic changed after deployment" in result.stderr - - -def test_container_context_excludes_local_environments_and_unrelated_projects(): - ignore_rules = DOCKERIGNORE.read_text(encoding="utf-8") - gitignore_rules = GITIGNORE.read_text(encoding="utf-8") - - assert "gha-creds-*.json" in ignore_rules - assert "gha-creds-*.json" in gitignore_rules - assert "**/.venv" in ignore_rules - assert "projects/*" in ignore_rules - assert "!projects/policyengine-simulation-entry/**" in ignore_rules - assert "projects/policyengine-simulation-entry/authenticated_tests" in ignore_rules - assert "libs/*" in ignore_rules - assert ignore_rules.index("**/.venv") > ignore_rules.index( - "!projects/policyengine-simulation-entry/**" - ) - assert ignore_rules.index("**/.venv") > ignore_rules.index( - "!libs/policyengine-simulation-contract/**" - ) - - -def test_production_lock_contains_the_direct_stage12_database_runtime(): - lockfile = tomllib.loads( - ( - REPOSITORY_ROOT / "projects" / "policyengine-simulation-entry" / "uv.lock" - ).read_text(encoding="utf-8") - ) - resolved_packages = {package["name"] for package in lockfile["package"]} - - assert { - "httpx", - "modal", - "policyengine-stage12-persistence", - "psycopg", - "sqlalchemy", - } <= resolved_packages - for package in ("policyengine-fastapi", "sqlmodel"): - assert package not in resolved_packages - - -def test_client_publication_checks_out_the_deployed_commit(): - workflow = PUBLISH_WORKFLOW.read_text(encoding="utf-8") - - assert "github.event.workflow_run.head_sha" in workflow - - -# The gateway's real /versions response for two published releases (6.2.7 -# latest); test_versions_fixture.py in the gateway project keeps it current. -VERSIONS_BODY = json.loads( - (Path(__file__).parent / "fixtures" / "versions_body.json").read_text() -) - - -def _run_identity_smoke(tmp_path: Path, versions: dict, **expected: str): - import json - - revision = "policyengine-simulation-entry-00002-new" - state_file, revisions_file = _write_fake_smoke_curl(tmp_path, [revision]) - versions_file = tmp_path / "versions.json" - versions_file.write_text(json.dumps(versions), encoding="utf-8") - env = { - **_smoke_test_env(tmp_path, state_file, revisions_file), - "FAKE_CURL_VERSIONS": str(versions_file), - **expected, - } - return _run_bash( - SMOKE_SCRIPT, - "https://simulation.example.test", - revision, - check=False, - capture_output=True, - env=env, - ) - - -IDENTITY = { - "EXPECTED_IDENTITY": "required", - "EXPECTED_POLICYENGINE_VERSION": "6.2.7", - "EXPECTED_APP_NAME": "policyengine-simulation-py6-2-7", - "EXPECTED_UK_MODEL_VERSION": "2.126.8", - "EXPECTED_UK_DATASET_SHA256": "a" * 64, - "EXPECTED_LATEST": "true", -} - - -def test_smoke_accepts_the_gateway_identity_of_a_new_release(tmp_path: Path): - result = _run_identity_smoke(tmp_path, VERSIONS_BODY, **IDENTITY) - - assert result.returncode == 0, result.stderr - assert ( - "serves policyengine 6.2.7 from policyengine-simulation-py6-2-7, " - "policyengine-uk 2.126.8, UK dataset " + "a" * 64 + " as latest" - ) in result.stdout - - -def test_smoke_accepts_an_older_release_by_its_route(tmp_path: Path): - result = _run_identity_smoke( - tmp_path, - VERSIONS_BODY, - EXPECTED_IDENTITY="required", - EXPECTED_POLICYENGINE_VERSION="6.2.6", - EXPECTED_APP_NAME="policyengine-simulation-py6-2-6", - EXPECTED_UK_MODEL_VERSION="2.125.1", - EXPECTED_UK_DATASET_SHA256="5" * 64, - ) - - assert result.returncode == 0, result.stderr - - -def test_smoke_without_expectations_only_requires_versions_json(tmp_path: Path): - result = _run_identity_smoke(tmp_path, {}) - - assert result.returncode == 0, result.stderr - - -def test_smoke_rejects_latest_routing_to_another_version(tmp_path: Path): - import copy - - versions = copy.deepcopy(VERSIONS_BODY) - versions["policyengine"]["latest"] = "6.2.6" - versions["uk"]["latest"] = "2.125.1" - - result = _run_identity_smoke(tmp_path, versions, **IDENTITY) - - assert result.returncode == 1 - assert "latest routes to policyengine 6.2.6 and policyengine-uk 2.125.1" in ( - result.stderr - ) - - -def test_smoke_rejects_a_uk_latest_that_lags_the_wrapper(tmp_path: Path): - import copy - - versions = copy.deepcopy(VERSIONS_BODY) - versions["uk"]["latest"] = "2.125.1" - - result = _run_identity_smoke(tmp_path, versions, **IDENTITY) - - assert result.returncode == 1 - assert "latest routes to" in result.stderr - - -def test_smoke_rejects_a_release_without_a_route(tmp_path: Path): - import copy - - versions = copy.deepcopy(VERSIONS_BODY) - del versions["policyengine"]["6.2.7"] - - result = _run_identity_smoke(tmp_path, versions, **IDENTITY) - - assert result.returncode == 1 - assert "no route and UK bundle identity for policyengine 6.2.7" in result.stderr - - -def test_smoke_rejects_a_uk_model_routed_to_another_app(tmp_path: Path): - import copy - - versions = copy.deepcopy(VERSIONS_BODY) - versions["uk"]["2.126.8"] = "policyengine-simulation-py6-2-6" - - result = _run_identity_smoke(tmp_path, versions, **IDENTITY) - - assert result.returncode == 1 - assert ( - "policyengine-uk 2.126.8 routes to policyengine-simulation-py6-2-6, " - "expected policyengine-simulation-py6-2-7" - ) in result.stderr - - -def test_smoke_rejects_another_uk_dataset(tmp_path: Path): - result = _run_identity_smoke( - tmp_path, VERSIONS_BODY, **{**IDENTITY, "EXPECTED_UK_DATASET_SHA256": "b" * 64} - ) - - assert result.returncode == 1 - assert "UK dataset SHA-256" in result.stderr - - -def test_smoke_rejects_identity_expectations_without_a_version(tmp_path: Path): - result = _run_identity_smoke( - tmp_path, VERSIONS_BODY, EXPECTED_UK_DATASET_SHA256="a" * 64 - ) - - assert result.returncode == 2 - assert "EXPECTED_POLICYENGINE_VERSION is required" in result.stderr - - -def test_smoke_accepts_an_unrecorded_digest_only_when_none_is_served( - tmp_path: Path, -): - """A release published before digests were recorded serves none.""" - import copy - - versions = copy.deepcopy(VERSIONS_BODY) - del versions["bundles"]["6.2.7"]["uk"]["dataset_sha256"] - expected = {**IDENTITY, "EXPECTED_UK_DATASET_SHA256": "unrecorded"} - (tmp_path / "unrecorded").mkdir() - (tmp_path / "served").mkdir() - result = _run_identity_smoke(tmp_path / "unrecorded", versions, **expected) - assert result.returncode == 0, result.stderr - assert "UK dataset unrecorded as latest" in result.stdout - served = _run_identity_smoke(tmp_path / "served", VERSIONS_BODY, **expected) - assert served.returncode == 1 - assert "is " + "a" * 64 + ", expected unrecorded" in served.stderr - - -def test_smoke_rejects_worker_routes_to_an_old_app(tmp_path: Path): - """#743 round-2 review: the version and its model route to the new app.""" - import copy - - versions = copy.deepcopy(VERSIONS_BODY) - versions["policyengine"]["6.2.7"] = "policyengine-simulation-py6-2-6" - versions["uk"]["2.126.8"] = "policyengine-simulation-py6-2-6" - - result = _run_identity_smoke(tmp_path, versions, **IDENTITY) - - assert result.returncode == 1 - assert ( - "policyengine 6.2.7 routes to policyengine-simulation-py6-2-6, " - "expected policyengine-simulation-py6-2-7" - ) in result.stderr - - -@pytest.mark.parametrize( - "missing", - [ - "EXPECTED_POLICYENGINE_VERSION", - "EXPECTED_APP_NAME", - "EXPECTED_UK_MODEL_VERSION", - "EXPECTED_UK_DATASET_SHA256", - ], -) -@pytest.mark.parametrize("how", ["unset", "empty"]) -def test_smoke_refuses_an_incomplete_identity(tmp_path: Path, missing, how): - """#743 round-2 review: an absent expectation never weakens the gate.""" - expected = dict(IDENTITY) - if how == "unset": - del expected[missing] - else: - expected[missing] = "" - - result = _run_identity_smoke(tmp_path, VERSIONS_BODY, **expected) - - assert result.returncode == 2 - assert f"{missing} is required" in result.stderr - - -def test_smoke_requires_every_expectation_when_identity_is_required(tmp_path: Path): - result = _run_identity_smoke(tmp_path, VERSIONS_BODY, EXPECTED_IDENTITY="required") - - assert result.returncode == 2 - assert "is required" in result.stderr - - -@pytest.mark.parametrize( - ("name", "value"), - [ - ("EXPECTED_IDENTITY", "optional"), - ("EXPECTED_LATEST", "yes"), - ("EXPECTED_UK_DATASET_SHA256", "A" * 64), - ("EXPECTED_UK_DATASET_SHA256", "a" * 63), - ], -) -def test_smoke_refuses_malformed_expectations(tmp_path: Path, name, value): - result = _run_identity_smoke(tmp_path, VERSIONS_BODY, **{**IDENTITY, name: value}) - - assert result.returncode == 2 - - -def test_smoke_rejects_a_bundle_recorded_for_another_wrapper(tmp_path: Path): - """#743 round-2 review: the bundle must be the expected version's own.""" - import copy - - versions = copy.deepcopy(VERSIONS_BODY) - versions["bundles"]["6.2.7"]["uk"]["policyengine_version"] = "6.2.6" - - result = _run_identity_smoke(tmp_path, versions, **IDENTITY) - - assert result.returncode == 1 - assert "no route and UK bundle identity for policyengine 6.2.7" in result.stderr - - -def _workflow_jobs(path: Path) -> dict: - import yaml - - return yaml.safe_load(path.read_text(encoding="utf-8"))["jobs"] - - -def test_deploy_gates_pass_the_release_identity_to_the_smoke(): - """Review M2: the active workflow (not comments) wires the whole identity.""" - jobs = _workflow_jobs(REUSABLE_DEPLOY_WORKFLOW) - assert ( - jobs["prepare"]["outputs"]["uk_dataset_sha256"] - == "${{ steps.versions.outputs.uk_dataset_sha256 }}" - ) - assert ( - jobs["deploy_executor"]["outputs"]["app_name"] - == "${{ steps.app.outputs.name }}" - ) - smoke_step = next( - step - for step in jobs["integration"]["steps"] - if "cloud-run-simulation-entry-smoke.sh" in step.get("run", "") - ) - expected = { - "EXPECTED_IDENTITY": ( - "${{ inputs.deploy_existing_stack && 'required' || 'none' }}" - ), - "EXPECTED_APP_NAME": ( - "${{ inputs.deploy_existing_stack && " - "needs.deploy_executor.outputs.app_name || '' }}" - ), - "EXPECTED_POLICYENGINE_VERSION": ( - "${{ inputs.deploy_existing_stack && " - "needs.prepare.outputs.policyengine_version || '' }}" - ), - "EXPECTED_UK_MODEL_VERSION": ( - "${{ inputs.deploy_existing_stack && " - "needs.prepare.outputs.uk_version || '' }}" - ), - "EXPECTED_UK_DATASET_SHA256": ( - "${{ inputs.deploy_existing_stack && " - "needs.prepare.outputs.uk_dataset_sha256 || '' }}" - ), - # Latest must route to this release only when it should advance it. - "EXPECTED_LATEST": ( - "${{ inputs.deploy_existing_stack && " - "needs.update_routing.outputs.advances_latest == 'true' && 'true' || '' }}" - ), - } - promote = jobs["promote_entrypoint"] - for environment in (smoke_step["env"], promote["env"]): - assert {key: environment.get(key) for key in expected} == expected - for job in ("integration", "promote_entrypoint"): - assert {"prepare", "deploy_executor", "update_routing"} <= set( - jobs[job]["needs"] - ) - # The promotion smoke runs inherit the job environment; no step overrides it. - for step in promote["steps"]: - if "cloud-run-simulation-entry-smoke.sh" in step.get("run", ""): - assert not set(step.get("env") or {}) & set(expected) - updates = _workflow_jobs( - REPOSITORY_ROOT / ".github" / "workflows" / "check-policyengine-updates.yml" - ) - update_envs = [ - step.get("env") or {} - for job in updates.values() - for step in job.get("steps", []) - ] - assert any( - env.get("RELEASE_PAYLOAD") == "${{ toJSON(github.event.client_payload) }}" - for env in update_envs - ) - - -def test_smoke_rejects_another_uk_model(tmp_path: Path): - result = _run_identity_smoke( - tmp_path, VERSIONS_BODY, **{**IDENTITY, "EXPECTED_UK_MODEL_VERSION": "2.125.1"} - ) - - assert result.returncode == 1 - assert "records policyengine-uk 2.126.8, expected 2.125.1" in result.stderr - - -def test_smoke_rejects_a_missing_bundle_identity(tmp_path: Path): - import copy - - versions = copy.deepcopy(VERSIONS_BODY) - versions["bundles"] = {} - - result = _run_identity_smoke(tmp_path, versions, **IDENTITY) - - assert result.returncode == 1 - assert "no route and UK bundle identity for policyengine 6.2.7" in result.stderr diff --git a/projects/policyengine-simulation-entry/tests/test_deployment_assets_cloud_run.py b/projects/policyengine-simulation-entry/tests/test_deployment_assets_cloud_run.py new file mode 100644 index 000000000..1052bfab5 --- /dev/null +++ b/projects/policyengine-simulation-entry/tests/test_deployment_assets_cloud_run.py @@ -0,0 +1,280 @@ +"""Tests for the Cloud Run deployment's environment, permissions and revision tags.""" + +from __future__ import annotations + +import json +import os +import re +import subprocess +import textwrap +from pathlib import Path + +import pytest +from test_deployment_assets_support import ( + CLOUD_RUN_DEPLOY_SCRIPT, + DEPLOY_WORKFLOW, + DOCKERFILE, + REUSABLE_DEPLOY_WORKFLOW, + STAGE12_DEPLOY_WORKFLOW, +) + + +def test_cloud_run_deployment_does_not_interpolate_github_values_in_shell(): + reusable_workflow = REUSABLE_DEPLOY_WORKFLOW.read_text(encoding="utf-8") + deployment_script = CLOUD_RUN_DEPLOY_SCRIPT.read_text(encoding="utf-8") + step_start = reusable_workflow.index( + " - name: Deploy Cloud Run entrypoint candidate" + ) + step_end = reusable_workflow.index( + "\n - name: Resolve exact candidate metadata", step_start + ) + deployment_step = reusable_workflow[step_start:step_end] + run_block = deployment_step[deployment_step.index("\n run:") :] + + assert "${{" not in run_block + assert ".github/scripts/deploy-cloud-run-simulation-entry.sh" in run_block + assert "--env-vars-file" in deployment_script + assert "--set-env-vars" not in deployment_script + assert "gcloud secrets versions list" in deployment_script + assert ":latest" not in deployment_script + + +def _cloud_run_deploy_environment(tmp_path: Path) -> dict[str, str]: + """Return a complete deployment environment backed by a fake gcloud.""" + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + fake_gcloud = fake_bin / "gcloud" + fake_gcloud.write_text( + textwrap.dedent( + """\ + #!/usr/bin/env bash + set -euo pipefail + + if [[ "$1 $2 $3" == "secrets versions list" ]]; then + printf '%s\n' 1 3 2 + exit 0 + fi + + if [[ "$1 $2" == "run deploy" ]]; then + printf '%s\n' "$@" > "${GCLOUD_ARGUMENTS_CAPTURE}" + while (($#)); do + if [[ "$1" == "--env-vars-file" ]]; then + cp "$2" "${GCLOUD_ENV_FILE_CAPTURE}" + exit 0 + fi + shift + done + fi + + printf 'unexpected gcloud invocation\n' >&2 + exit 1 + """ + ), + encoding="utf-8", + ) + fake_gcloud.chmod(0o755) + environment = { + name: value + for name, value in os.environ.items() + if name != "SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS_VALUE" + } + return environment | { + "PATH": f"{fake_bin}:{os.environ['PATH']}", + "RUNNER_TEMP": str(tmp_path), + "GCLOUD_ARGUMENTS_CAPTURE": str(tmp_path / "gcloud-arguments.txt"), + "GCLOUD_ENV_FILE_CAPTURE": str(tmp_path / "runtime-environment.json"), + "PROJECT_ID": "policyengine-staging", + "REGION": "us-central1", + "IMAGE": "us-central1-docker.pkg.dev/project/repository/image:sha", + "TAG": "s-123", + "DEPLOY_STAGE12_V2": "true", + "APP_ENVIRONMENT": "staging", + "MODAL_ENVIRONMENT": "staging", + "ENTRYPOINT_SERVICE": "policyengine-simulation-entry-staging", + "ENTRYPOINT_RUNTIME_SERVICE_ACCOUNT": ( + "simulation-entry@policyengine-staging.iam.gserviceaccount.com" + ), + "ENTRYPOINT_MIN_INSTANCES": "0", + "ENTRYPOINT_MAX_INSTANCES": "10", + "SIMULATION_ENTRYPOINT_AUTH_ISSUER_VALUE": ( + "https://issuer.example/path?literal=$(not-a-command)" + ), + "SIMULATION_ENTRYPOINT_AUTH_AUDIENCE_VALUE": "simulation-api", + "OLD_GATEWAY_URL_VALUE": "https://legacy.example", + "OLD_GATEWAY_AUTH_ISSUER_VALUE": "https://issuer.example", + "OLD_GATEWAY_AUTH_AUDIENCE_VALUE": "legacy-api", + "OLD_GATEWAY_AUTH_CLIENT_ID_VALUE": "entrypoint-client", + "OLD_GATEWAY_AUTH_CLIENT_SECRET_SECRET_NAME": "old-client-secret", + "OBSERVABILITY_SERVICE_NAMESPACE": "policyengine.simulation-entry", + "OBSERVABILITY_TRACE_PROJECT_ID": "policyengine-observability", + "OTEL_EXPORTER_OTLP_ENDPOINT": "https://collector.example", + "POLICYENGINE_OTEL_GOOGLE_AUDIENCE": "https://collector.example", + "STAGE12_ENABLED_VALUE": "0", + "STAGE12_ARTIFACT_BUCKET_VALUE": "policyengine-stage12-staging", + "STAGE12_DATABASE_URL_SECRET_NAME": "stage12-staging-database-url", + "MODAL_TOKEN_ID_SECRET_NAME": "modal-token-id", + "MODAL_TOKEN_SECRET_SECRET_NAME": "modal-token-secret", + } + + +def test_cloud_run_deployment_escapes_environment_and_pins_secret_versions(tmp_path): + command_environment = _cloud_run_deploy_environment(tmp_path) + arguments_capture = Path(command_environment["GCLOUD_ARGUMENTS_CAPTURE"]) + env_capture = Path(command_environment["GCLOUD_ENV_FILE_CAPTURE"]) + + subprocess.run( + [CLOUD_RUN_DEPLOY_SCRIPT], + check=True, + env=command_environment, + ) + + runtime_environment = json.loads(env_capture.read_text(encoding="utf-8")) + assert runtime_environment["SIMULATION_ENTRYPOINT_AUTH_ISSUER"] == ( + "https://issuer.example/path?literal=$(not-a-command)" + ) + assert runtime_environment["STAGE12_ENABLED"] == "0" + assert runtime_environment["STAGE12_V2_MANIFEST_ENVIRONMENT"] == "staging" + assert runtime_environment["OBSERVABILITY_SERVICE_NAMESPACE"] == ( + "policyengine.simulation-entry" + ) + assert runtime_environment["OBSERVABILITY_TRACE_PROJECT_ID"] == ( + "policyengine-observability" + ) + assert runtime_environment["CLOUD_RUN_REGION"] == "us-central1" + assert runtime_environment["OTEL_EXPORTER_OTLP_ENDPOINT"] == ( + "https://collector.example" + ) + assert runtime_environment["POLICYENGINE_OTEL_GOOGLE_AUDIENCE"] == ( + "https://collector.example" + ) + assert "STAGE12_DATABASE_URL" not in runtime_environment + arguments = arguments_capture.read_text(encoding="utf-8").splitlines() + secrets_argument = arguments[arguments.index("--set-secrets") + 1] + assert secrets_argument == ( + "OLD_GATEWAY_AUTH_CLIENT_SECRET=old-client-secret:3," + "MODAL_TOKEN_ID=modal-token-id:3," + "MODAL_TOKEN_SECRET=modal-token-secret:3," + "STAGE12_DATABASE_URL=stage12-staging-database-url:3" + ) + assert ":latest" not in secrets_argument + + +@pytest.mark.parametrize( + ("configured", "expected"), + [(None, "0"), ("", "0"), ("0", "0"), ("1", "1")], + ids=["unset", "empty", "log-only", "enforced"], +) +def test_cloud_run_deployment_defaults_permission_enforcement_to_log_only( + tmp_path, + configured, + expected, +): + command_environment = _cloud_run_deploy_environment(tmp_path) + if configured is not None: + command_environment["SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS_VALUE"] = ( + configured + ) + + subprocess.run( + [CLOUD_RUN_DEPLOY_SCRIPT], + check=True, + env=command_environment, + ) + + runtime_environment = json.loads( + Path(command_environment["GCLOUD_ENV_FILE_CAPTURE"]).read_text(encoding="utf-8") + ) + assert runtime_environment["SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS"] == expected + + +@pytest.mark.parametrize("configured", ["true", "yes", "2", " 1"]) +def test_cloud_run_deployment_rejects_invalid_permission_enforcement( + tmp_path, + configured, +): + command_environment = _cloud_run_deploy_environment(tmp_path) + command_environment["SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS_VALUE"] = configured + + result = subprocess.run( + [CLOUD_RUN_DEPLOY_SCRIPT], + check=False, + env=command_environment, + capture_output=True, + text=True, + ) + + assert result.returncode != 0 + assert ( + "SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS_VALUE must be 0 or 1" + in result.stderr + ) + assert not Path(command_environment["GCLOUD_ENV_FILE_CAPTURE"]).exists() + + +def test_permission_enforcement_reaches_candidate_and_authenticated_tests(): + reusable_workflow = REUSABLE_DEPLOY_WORKFLOW.read_text(encoding="utf-8") + + assert ( + reusable_workflow.count( + "SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS_VALUE: " + "${{ vars.SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS }}" + ) + == 1 + ) + # Every deployed authentication run must know whether the target revision + # enforces permissions, so its enforcement check runs or skips correctly. + authenticated_runs = reusable_workflow.count( + "run: uv run --frozen pytest authenticated_tests/ -v" + ) + assert authenticated_runs == 3 + assert ( + reusable_workflow.count( + "SIMULATION_ENTRYPOINT_TEST_ENFORCE_PERMISSIONS: " + "${{ vars.SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS }}" + ) + == authenticated_runs + ) + + +def test_cloud_run_revision_tags_are_valid_on_the_first_workflow_run(): + deploy_workflow = DEPLOY_WORKFLOW.read_text(encoding="utf-8") + reusable_workflow = REUSABLE_DEPLOY_WORKFLOW.read_text(encoding="utf-8") + prefixes = re.findall( + r"^\s+entrypoint_revision_prefix:\s+(\S+)$", + deploy_workflow, + flags=re.MULTILINE, + ) + + assert ( + "tag=${{ inputs.entrypoint_revision_prefix }}${GITHUB_RUN_NUMBER}" + in reusable_workflow + ) + assert len(prefixes) == 2 + + first_run_tags = [f"{prefix}1" for prefix in prefixes] + for tag in first_run_tags: + assert 3 <= len(tag) <= 63 + assert re.fullmatch(r"[a-z][a-z0-9-]*[a-z0-9]", tag) + + +def test_revision_tags_fit_existing_cloud_run_service_names(): + deploy_workflow = DEPLOY_WORKFLOW.read_text(encoding="utf-8") + stage12_workflow = STAGE12_DEPLOY_WORKFLOW.read_text(encoding="utf-8") + deployments = { + "policyengine-simulation-entry-staging": "s-", + "policyengine-simulation-entry": "p-", + } + + for workflow in (deploy_workflow, stage12_workflow): + for service, prefix in deployments.items(): + assert f"entrypoint_service: {service}" in workflow + assert f"entrypoint_revision_prefix: {prefix}" in workflow + # Reserve seven digits for the monotonically increasing workflow + # run number while satisfying Cloud Run's service-plus-tag limit. + assert len(service) + len(f"{prefix}9999999") <= 46 + + +def test_entrypoint_container_uses_requested_target_platform(): + dockerfile = DOCKERFILE.read_text(encoding="utf-8") + + assert "$BUILDPLATFORM" not in dockerfile diff --git a/projects/policyengine-simulation-entry/tests/test_deployment_assets_identity_smoke.py b/projects/policyengine-simulation-entry/tests/test_deployment_assets_identity_smoke.py new file mode 100644 index 000000000..1e833139d --- /dev/null +++ b/projects/policyengine-simulation-entry/tests/test_deployment_assets_identity_smoke.py @@ -0,0 +1,303 @@ +"""Tests for the smoke check of the release identity the gateway serves.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest +from test_deployment_assets_support import ( + IDENTITY, + REPOSITORY_ROOT, + REUSABLE_DEPLOY_WORKFLOW, + VERSIONS_BODY, + run_identity_smoke, + workflow_jobs, +) + +if TYPE_CHECKING: + from pathlib import Path + + +def test_smoke_accepts_the_gateway_identity_of_a_new_release(tmp_path: Path): + result = run_identity_smoke(tmp_path, VERSIONS_BODY, **IDENTITY) + + assert result.returncode == 0, result.stderr + assert ( + "serves policyengine 6.2.7 from policyengine-simulation-py6-2-7, " + "policyengine-uk 2.126.8, UK dataset " + "a" * 64 + " as latest" + ) in result.stdout + + +def test_smoke_accepts_an_older_release_by_its_route(tmp_path: Path): + result = run_identity_smoke( + tmp_path, + VERSIONS_BODY, + EXPECTED_IDENTITY="required", + EXPECTED_POLICYENGINE_VERSION="6.2.6", + EXPECTED_APP_NAME="policyengine-simulation-py6-2-6", + EXPECTED_UK_MODEL_VERSION="2.125.1", + EXPECTED_UK_DATASET_SHA256="5" * 64, + ) + + assert result.returncode == 0, result.stderr + + +def test_smoke_without_expectations_only_requires_versions_json(tmp_path: Path): + result = run_identity_smoke(tmp_path, {}) + + assert result.returncode == 0, result.stderr + + +def test_smoke_rejects_latest_routing_to_another_version(tmp_path: Path): + import copy + + versions = copy.deepcopy(VERSIONS_BODY) + versions["policyengine"]["latest"] = "6.2.6" + versions["uk"]["latest"] = "2.125.1" + + result = run_identity_smoke(tmp_path, versions, **IDENTITY) + + assert result.returncode == 1 + assert "latest routes to policyengine 6.2.6 and policyengine-uk 2.125.1" in ( + result.stderr + ) + + +def test_smoke_rejects_a_uk_latest_that_lags_the_wrapper(tmp_path: Path): + import copy + + versions = copy.deepcopy(VERSIONS_BODY) + versions["uk"]["latest"] = "2.125.1" + + result = run_identity_smoke(tmp_path, versions, **IDENTITY) + + assert result.returncode == 1 + assert "latest routes to" in result.stderr + + +def test_smoke_rejects_a_release_without_a_route(tmp_path: Path): + import copy + + versions = copy.deepcopy(VERSIONS_BODY) + del versions["policyengine"]["6.2.7"] + + result = run_identity_smoke(tmp_path, versions, **IDENTITY) + + assert result.returncode == 1 + assert "no route and UK bundle identity for policyengine 6.2.7" in result.stderr + + +def test_smoke_rejects_a_uk_model_routed_to_another_app(tmp_path: Path): + import copy + + versions = copy.deepcopy(VERSIONS_BODY) + versions["uk"]["2.126.8"] = "policyengine-simulation-py6-2-6" + + result = run_identity_smoke(tmp_path, versions, **IDENTITY) + + assert result.returncode == 1 + assert ( + "policyengine-uk 2.126.8 routes to policyengine-simulation-py6-2-6, " + "expected policyengine-simulation-py6-2-7" + ) in result.stderr + + +def test_smoke_rejects_another_uk_dataset(tmp_path: Path): + result = run_identity_smoke( + tmp_path, VERSIONS_BODY, **{**IDENTITY, "EXPECTED_UK_DATASET_SHA256": "b" * 64} + ) + + assert result.returncode == 1 + assert "UK dataset SHA-256" in result.stderr + + +def test_smoke_rejects_identity_expectations_without_a_version(tmp_path: Path): + result = run_identity_smoke( + tmp_path, VERSIONS_BODY, EXPECTED_UK_DATASET_SHA256="a" * 64 + ) + + assert result.returncode == 2 + assert "EXPECTED_POLICYENGINE_VERSION is required" in result.stderr + + +def test_smoke_accepts_an_unrecorded_digest_only_when_none_is_served( + tmp_path: Path, +): + """A release published before digests were recorded serves none.""" + import copy + + versions = copy.deepcopy(VERSIONS_BODY) + del versions["bundles"]["6.2.7"]["uk"]["dataset_sha256"] + expected = {**IDENTITY, "EXPECTED_UK_DATASET_SHA256": "unrecorded"} + (tmp_path / "unrecorded").mkdir() + (tmp_path / "served").mkdir() + result = run_identity_smoke(tmp_path / "unrecorded", versions, **expected) + assert result.returncode == 0, result.stderr + assert "UK dataset unrecorded as latest" in result.stdout + served = run_identity_smoke(tmp_path / "served", VERSIONS_BODY, **expected) + assert served.returncode == 1 + assert "is " + "a" * 64 + ", expected unrecorded" in served.stderr + + +def test_smoke_rejects_worker_routes_to_an_old_app(tmp_path: Path): + """#743 round-2 review: the version and its model route to the new app.""" + import copy + + versions = copy.deepcopy(VERSIONS_BODY) + versions["policyengine"]["6.2.7"] = "policyengine-simulation-py6-2-6" + versions["uk"]["2.126.8"] = "policyengine-simulation-py6-2-6" + + result = run_identity_smoke(tmp_path, versions, **IDENTITY) + + assert result.returncode == 1 + assert ( + "policyengine 6.2.7 routes to policyengine-simulation-py6-2-6, " + "expected policyengine-simulation-py6-2-7" + ) in result.stderr + + +@pytest.mark.parametrize( + "missing", + [ + "EXPECTED_POLICYENGINE_VERSION", + "EXPECTED_APP_NAME", + "EXPECTED_UK_MODEL_VERSION", + "EXPECTED_UK_DATASET_SHA256", + ], +) +@pytest.mark.parametrize("how", ["unset", "empty"]) +def test_smoke_refuses_an_incomplete_identity(tmp_path: Path, missing, how): + """#743 round-2 review: an absent expectation never weakens the gate.""" + expected = dict(IDENTITY) + if how == "unset": + del expected[missing] + else: + expected[missing] = "" + + result = run_identity_smoke(tmp_path, VERSIONS_BODY, **expected) + + assert result.returncode == 2 + assert f"{missing} is required" in result.stderr + + +def test_smoke_requires_every_expectation_when_identity_is_required(tmp_path: Path): + result = run_identity_smoke(tmp_path, VERSIONS_BODY, EXPECTED_IDENTITY="required") + + assert result.returncode == 2 + assert "is required" in result.stderr + + +@pytest.mark.parametrize( + ("name", "value"), + [ + ("EXPECTED_IDENTITY", "optional"), + ("EXPECTED_LATEST", "yes"), + ("EXPECTED_UK_DATASET_SHA256", "A" * 64), + ("EXPECTED_UK_DATASET_SHA256", "a" * 63), + ], +) +def test_smoke_refuses_malformed_expectations(tmp_path: Path, name, value): + result = run_identity_smoke(tmp_path, VERSIONS_BODY, **{**IDENTITY, name: value}) + + assert result.returncode == 2 + + +def test_smoke_rejects_a_bundle_recorded_for_another_wrapper(tmp_path: Path): + """#743 round-2 review: the bundle must be the expected version's own.""" + import copy + + versions = copy.deepcopy(VERSIONS_BODY) + versions["bundles"]["6.2.7"]["uk"]["policyengine_version"] = "6.2.6" + + result = run_identity_smoke(tmp_path, versions, **IDENTITY) + + assert result.returncode == 1 + assert "no route and UK bundle identity for policyengine 6.2.7" in result.stderr + + +def test_deploy_gates_pass_the_release_identity_to_the_smoke(): + """Review M2: the active workflow (not comments) wires the whole identity.""" + jobs = workflow_jobs(REUSABLE_DEPLOY_WORKFLOW) + assert ( + jobs["prepare"]["outputs"]["uk_dataset_sha256"] + == "${{ steps.versions.outputs.uk_dataset_sha256 }}" + ) + assert ( + jobs["deploy_executor"]["outputs"]["app_name"] + == "${{ steps.app.outputs.name }}" + ) + smoke_step = next( + step + for step in jobs["integration"]["steps"] + if "cloud-run-simulation-entry-smoke.sh" in step.get("run", "") + ) + expected = { + "EXPECTED_IDENTITY": ( + "${{ inputs.deploy_existing_stack && 'required' || 'none' }}" + ), + "EXPECTED_APP_NAME": ( + "${{ inputs.deploy_existing_stack && " + "needs.deploy_executor.outputs.app_name || '' }}" + ), + "EXPECTED_POLICYENGINE_VERSION": ( + "${{ inputs.deploy_existing_stack && " + "needs.prepare.outputs.policyengine_version || '' }}" + ), + "EXPECTED_UK_MODEL_VERSION": ( + "${{ inputs.deploy_existing_stack && " + "needs.prepare.outputs.uk_version || '' }}" + ), + "EXPECTED_UK_DATASET_SHA256": ( + "${{ inputs.deploy_existing_stack && " + "needs.prepare.outputs.uk_dataset_sha256 || '' }}" + ), + # Latest must route to this release only when it should advance it. + "EXPECTED_LATEST": ( + "${{ inputs.deploy_existing_stack && " + "needs.update_routing.outputs.advances_latest == 'true' && 'true' || '' }}" + ), + } + promote = jobs["promote_entrypoint"] + for environment in (smoke_step["env"], promote["env"]): + assert {key: environment.get(key) for key in expected} == expected + for job in ("integration", "promote_entrypoint"): + assert {"prepare", "deploy_executor", "update_routing"} <= set( + jobs[job]["needs"] + ) + # The promotion smoke runs inherit the job environment; no step overrides it. + for step in promote["steps"]: + if "cloud-run-simulation-entry-smoke.sh" in step.get("run", ""): + assert not set(step.get("env") or {}) & set(expected) + updates = workflow_jobs( + REPOSITORY_ROOT / ".github" / "workflows" / "check-policyengine-updates.yml" + ) + update_envs = [ + step.get("env") or {} + for job in updates.values() + for step in job.get("steps", []) + ] + assert any( + env.get("RELEASE_PAYLOAD") == "${{ toJSON(github.event.client_payload) }}" + for env in update_envs + ) + + +def test_smoke_rejects_another_uk_model(tmp_path: Path): + result = run_identity_smoke( + tmp_path, VERSIONS_BODY, **{**IDENTITY, "EXPECTED_UK_MODEL_VERSION": "2.125.1"} + ) + + assert result.returncode == 1 + assert "records policyengine-uk 2.126.8, expected 2.125.1" in result.stderr + + +def test_smoke_rejects_a_missing_bundle_identity(tmp_path: Path): + import copy + + versions = copy.deepcopy(VERSIONS_BODY) + versions["bundles"] = {} + + result = run_identity_smoke(tmp_path, versions, **IDENTITY) + + assert result.returncode == 1 + assert "no route and UK bundle identity for policyengine 6.2.7" in result.stderr diff --git a/projects/policyengine-simulation-entry/tests/test_deployment_assets_promotion.py b/projects/policyengine-simulation-entry/tests/test_deployment_assets_promotion.py new file mode 100644 index 000000000..30f3b9eda --- /dev/null +++ b/projects/policyengine-simulation-entry/tests/test_deployment_assets_promotion.py @@ -0,0 +1,269 @@ +"""Tests for promotion order, Stage 12-only deployments and the sentinel.""" + +from __future__ import annotations + +import re + +from test_deployment_assets_support import ( + DEPLOY_WORKFLOW, + DEPLOYMENT_READY_SCRIPT, + REUSABLE_DEPLOY_WORKFLOW, + STAGE12_DEPLOY_WORKFLOW, + job_section, + sentinel_check_step, + workflow_jobs, +) + + +def test_full_stack_promotion_order_is_explicit(): + deploy_workflow = DEPLOY_WORKFLOW.read_text(encoding="utf-8") + reusable_workflow = REUSABLE_DEPLOY_WORKFLOW.read_text(encoding="utf-8") + + for deployment_step, unit_test_step in ( + ("Deploy Cloud Run entrypoint candidate", "Run entrypoint unit tests"), + ("Deploy stable Modal gateway", "Run gateway unit tests"), + ("Deploy versioned Modal executor", "Run executor unit tests"), + ): + assert reusable_workflow.index(deployment_step) < reusable_workflow.index( + unit_test_step + ) + + routing_dependencies = ( + "needs: [prepare, deploy_entrypoint, deploy_gateway, deploy_executor]" + ) + assert routing_dependencies in reusable_workflow + gateway_job = reusable_workflow[ + reusable_workflow.index("\n deploy_gateway:") : reusable_workflow.index( + "\n deploy_executor:" + ) + ] + assert "needs: [prepare, deploy_executor]" in gateway_job + assert reusable_workflow.index("\n update_routing:") < reusable_workflow.index( + "\n integration:" + ) + assert ( + "stage12_v2_ready" + in reusable_workflow[ + reusable_workflow.index("\n integration:") : reusable_workflow.index( + "\n authenticated_test:" + ) + ] + ) + assert reusable_workflow.index("\n integration:") < reusable_workflow.index( + "\n authenticated_test:" + ) + assert reusable_workflow.index("\n authenticated_test:") < reusable_workflow.index( + "\n promote_entrypoint:" + ) + promote_needs = set( + workflow_jobs(REUSABLE_DEPLOY_WORKFLOW)["promote_entrypoint"]["needs"] + ) + assert { + "prepare", + "deploy_entrypoint", + "deploy_executor", + "authenticated_test", + "sentinel_expected", + "update_routing", + } <= promote_needs + assert ( + 'cloud-run-simulation-entry-smoke.sh "${STABLE_URL}" "${TARGET_REVISION}" ' + '"${PREVIOUS_REVISION}"' in reusable_workflow + ) + assert ( + 'cloud-run-simulation-entry-smoke.sh "${CUSTOM_URL}" "${TARGET_REVISION}" ' + '"${PREVIOUS_REVISION}"' in reusable_workflow + ) + assert ( + "cloud-run-simulation-entry-smoke.sh " + '"${{ needs.deploy_entrypoint.outputs.candidate_url }}" ' + '"${{ needs.deploy_entrypoint.outputs.revision }}"' in reusable_workflow + ) + assert "failure() && steps.promote.outcome == 'success'" in reusable_workflow + assert "needs: beta" in deploy_workflow + assert "skip_beta" not in deploy_workflow + + +def test_stage12_only_deployment_runs_candidate_tests_and_requested_promotion(): + reusable_workflow = REUSABLE_DEPLOY_WORKFLOW.read_text(encoding="utf-8") + + authenticated_test = re.search( + r"^ authenticated_test:\n(?P.*?)(?=^ [a-z0-9_-]+:\n|\Z)", + reusable_workflow, + flags=re.MULTILINE | re.DOTALL, + ) + assert authenticated_test is not None + authenticated_test_body = authenticated_test.group("body") + assert "!cancelled()" in authenticated_test_body + assert "needs.deploy_entrypoint.result == 'success'" in authenticated_test_body + assert "needs.integration.result == 'success'" in authenticated_test_body + + promotion = re.search( + r"^ promote_entrypoint:\n(?P.*?)(?=^ [a-z0-9_-]+:\n|\Z)", + reusable_workflow, + flags=re.MULTILINE | re.DOTALL, + ) + assert promotion is not None + promotion_body = promotion.group("body") + assert "!cancelled()" in promotion_body + assert "inputs.promote_entrypoint" in promotion_body + assert "needs.deploy_entrypoint.result == 'success'" in promotion_body + assert "needs.authenticated_test.result == 'success'" in promotion_body + + +def test_reusable_deployment_rejects_unexpected_skipped_phases(): + reusable_workflow = REUSABLE_DEPLOY_WORKFLOW.read_text(encoding="utf-8") + completion = re.search( + r"^ deployment_ready:\n(?P.*?)(?=^ [a-z0-9_-]+:\n|\Z)", + reusable_workflow, + flags=re.MULTILINE | re.DOTALL, + ) + + assert completion is not None + completion_body = completion.group("body") + assert "if: ${{ !cancelled() }}" in completion_body + for dependency in ( + "deploy_entrypoint", + "integration", + "authenticated_test", + "promote_entrypoint", + ): + assert dependency in completion_body + assert "run: .github/scripts/simulation-deployment-ready.sh" in completion_body + for result in ( + "ENTRYPOINT_RESULT: ${{ needs.deploy_entrypoint.result }}", + "INTEGRATION_RESULT: ${{ needs.integration.result }}", + "AUTHENTICATED_TEST_RESULT: ${{ needs.authenticated_test.result }}", + "PROMOTION_RESULT: ${{ needs.promote_entrypoint.result }}", + ): + assert result in completion_body + readiness_check = DEPLOYMENT_READY_SCRIPT.read_text(encoding="utf-8") + for expected_result in ( + 'test "${ENTRYPOINT_RESULT}" = "success"', + 'test "${INTEGRATION_RESULT}" = "success"', + 'test "${AUTHENTICATED_TEST_RESULT}" = "success"', + 'test "${PROMOTION_RESULT}" = "success"', + 'test "${PROMOTION_RESULT}" = "skipped"', + ): + assert expected_result in readiness_check + + +def test_complete_integration_suite_is_configured_for_beta_only(): + deploy_workflow = DEPLOY_WORKFLOW.read_text(encoding="utf-8") + reusable_workflow = REUSABLE_DEPLOY_WORKFLOW.read_text(encoding="utf-8") + + beta_workflow = deploy_workflow[ + deploy_workflow.index(" beta:") : deploy_workflow.index(" prod:") + ] + prod_workflow = deploy_workflow[ + deploy_workflow.index(" prod:") : deploy_workflow.index(" summary:") + ] + + assert "run_full_integration: true" in beta_workflow + assert "run_full_integration: false" in prod_workflow + assert "run_full_integration:" in reusable_workflow + + +def test_stage12_only_deployment_skips_existing_stack_integration_suite(): + stage12_workflow = STAGE12_DEPLOY_WORKFLOW.read_text(encoding="utf-8") + + assert stage12_workflow.count("run_full_integration: false") == 2 + assert "run_full_integration: true" not in stage12_workflow + + +def test_main_deployment_automatically_deploys_stage12_in_both_environments(): + deploy_workflow = DEPLOY_WORKFLOW.read_text(encoding="utf-8") + reusable_workflow = REUSABLE_DEPLOY_WORKFLOW.read_text(encoding="utf-8") + + beta_workflow = deploy_workflow[ + deploy_workflow.index(" beta:") : deploy_workflow.index(" prod:") + ] + prod_workflow = deploy_workflow[ + deploy_workflow.index(" prod:") : deploy_workflow.index(" summary:") + ] + + assert "deploy_stage12_v2: true" in beta_workflow + assert "deploy_stage12_v2: true" in prod_workflow + assert deploy_workflow.count("deploy_stage12_v2: true") == 2 + assert "deploy_stage12_v2: false" not in deploy_workflow + + assert "src.modal.utils.set_stage12_dual_execution" not in reusable_workflow + assert "STAGE12_ENABLED_VALUE: ${{ vars.STAGE12_ENABLED }}" in reusable_workflow + + +def test_stage12_worker_deployment_can_request_a_google_oidc_token(): + reusable_workflow = REUSABLE_DEPLOY_WORKFLOW.read_text(encoding="utf-8") + match = re.search( + r"^ deploy_stage12_v2:\n(?P.*?)(?=^ [a-z0-9_-]+:\n|\Z)", + reusable_workflow, + flags=re.MULTILINE | re.DOTALL, + ) + + assert match is not None + section = match.group("body") + assert "permissions:\n contents: read\n id-token: write" in section + assert "google-github-actions/auth@v2" in section + + +def test_stage12_only_deployment_cannot_redeploy_existing_modal_resources(): + reusable_workflow = REUSABLE_DEPLOY_WORKFLOW.read_text(encoding="utf-8") + + for job_name in ("deploy_gateway", "deploy_executor", "update_routing"): + match = re.search( + rf"^ {job_name}:\n(?P.*?)(?=^ [a-z0-9_-]+:\n|\Z)", + reusable_workflow, + flags=re.MULTILINE | re.DOTALL, + ) + assert match is not None + section = match.group("body") + assert "if: ${{ inputs.deploy_existing_stack }}" in section + + stage12_section = reusable_workflow[ + reusable_workflow.index("\n deploy_stage12_v2:") : reusable_workflow.index( + "\n integration:" + ) + ] + assert "src.modal.utils.update_version_registry" not in stage12_section + finalizer_job = job_section(reusable_workflow, "finalize_routing") + assert "inputs.deploy_existing_stack &&" in finalizer_job + sentinel_job = job_section(reusable_workflow, "sentinel_expected") + assert "if: ${{ inputs.deploy_existing_stack }}" in sentinel_job + + +def test_deployed_stack_must_reproduce_the_numerical_sentinel(): + workflow = REUSABLE_DEPLOY_WORKFLOW.read_text(encoding="utf-8") + jobs = workflow_jobs(REUSABLE_DEPLOY_WORKFLOW) + sentinel_job = job_section(workflow, "sentinel_expected") + assert jobs["sentinel_expected"]["needs"] == "prepare" + assert "release_sentinel expected" in sentinel_job + assert "secrets.PE_UK_PRIVATE_HF_READ_TOKEN" in sentinel_job + assert "secrets.GCP_CREDENTIALS_JSON" in sentinel_job + assert ".uk.dataset_sha256 == $d" in sentinel_job + + integration = jobs["integration"] + assert "needs.sentinel_expected.result == 'success'" in integration["if"] + assert "needs.sentinel_expected.result == 'skipped'" in integration["if"] + check = sentinel_check_step(integration) + # Both checks run on every existing-stack deployment, after the identity + # smoke, against the candidate and then the stable URL. + assert check["if"] == "${{ inputs.deploy_existing_stack }}" + assert "needs.deploy_entrypoint.outputs.candidate_url" in check["run"] + steps = integration["steps"] + smoke = next( + index + for index, step in enumerate(steps) + if "cloud-run-simulation-entry-smoke.sh" in step.get("run", "") + ) + assert smoke < steps.index(check) + + promote = jobs["promote_entrypoint"] + stable = sentinel_check_step(promote) + assert stable["if"] == "${{ inputs.deploy_existing_stack }}" + assert '--base-url "${STABLE_URL}"' in stable["run"] + names = [step.get("name", "") for step in promote["steps"]] + # A failing stable sentinel triggers the existing traffic restore step. + assert promote["steps"].index(stable) < next( + index + for index, name in enumerate(names) + if name.startswith("Restore previous revision") + ) diff --git a/projects/policyengine-simulation-entry/tests/test_deployment_assets_release_gates.py b/projects/policyengine-simulation-entry/tests/test_deployment_assets_release_gates.py new file mode 100644 index 000000000..829aa6985 --- /dev/null +++ b/projects/policyengine-simulation-entry/tests/test_deployment_assets_release_gates.py @@ -0,0 +1,302 @@ +"""Tests for the expression reader, and the release checks that finalize routing.""" + +from __future__ import annotations + +import os +from typing import TYPE_CHECKING + +import pytest +from test_deployment_assets_support import ( + IDENTITY, + REUSABLE_DEPLOY_WORKFLOW, + VERSIONS_BODY, + evaluate_github_expression, + routing_context, + run_bash, + run_identity_smoke, + sentinel_check_step, + workflow_jobs, +) + +if TYPE_CHECKING: + from pathlib import Path + + +def test_the_expression_evaluator_follows_github_semantics(): + context = {"inputs": {"flag": True}, "needs": {"a": {"result": "failure"}}} + assert evaluate_github_expression("${{ inputs.flag == 'true' }}", context) is False + assert evaluate_github_expression("${{ inputs.flag == true }}", context) is True + assert evaluate_github_expression("${{ 'SUCCESS' == 'success' }}", context) is True + assert evaluate_github_expression("${{ inputs.missing == '' }}", context) is True + # No status function: implicitly success() first, so a failed need skips it. + assert not evaluate_github_expression("${{ inputs.flag }}", context, job_if=True) + assert evaluate_github_expression( + "${{ always() && inputs.flag }}", context, job_if=True + ) + assert not evaluate_github_expression( + "${{ !cancelled() }}", {**context, "cancelled": True} + ) + + +def test_the_expression_evaluator_returns_operands_and_job_statuses(): + context = { + "inputs": {"name": "beta", "empty": "", "zero": 0, "flag": False}, + "needs": {"a": {"result": "success"}, "b": {"result": "skipped"}}, + "steps": "not-a-mapping", + } + # || and && return one of their operands, as GitHub does, not a boolean. + assert ( + evaluate_github_expression("${{ inputs.empty || 'fallback' }}", context) + == "fallback" + ) + assert ( + evaluate_github_expression("${{ inputs.name || 'fallback' }}", context) + == "beta" + ) + assert evaluate_github_expression("${{ inputs.name && inputs.zero }}", context) == 0 + assert ( + evaluate_github_expression("${{ inputs.flag && inputs.name }}", context) + is False + ) + assert evaluate_github_expression("${{ inputs.name != 'BETA' }}", context) is False + assert ( + evaluate_github_expression("${{ !(inputs.empty || inputs.zero) }}", context) + is True + ) + assert evaluate_github_expression("${{ steps.x.outputs }}", context) is None + assert evaluate_github_expression("${{ inputs.missing.deeper }}", context) is None + assert evaluate_github_expression("false", context) is False + # success() needs every needed job to succeed; failure() needs one failure. + assert evaluate_github_expression("${{ success() }}", context) is False + assert evaluate_github_expression("${{ failure() }}", context) is False + assert evaluate_github_expression("${{ success() }}", {}) is True + assert evaluate_github_expression("${{ success() }}", {"cancelled": True}) is False + assert evaluate_github_expression("${{ cancelled() }}", {"cancelled": True}) is True + # Any status function in a job if replaces the implicit success(). + assert ( + evaluate_github_expression("${{ inputs.name == 'beta' }}", context, job_if=True) + is False + ) + assert ( + evaluate_github_expression( + "${{ inputs.name == 'beta' || failure() }}", context, job_if=True + ) + is True + ) + + +@pytest.mark.parametrize( + "expression", + ["${{ unknown() }}", "${{ inputs.name inputs.name }}", "${{ inputs.name @ }}"], + ids=["unknown-function", "trailing-token", "unknown-character"], +) +def test_the_expression_evaluator_refuses_what_it_cannot_read(expression): + with pytest.raises(AssertionError): + evaluate_github_expression(expression, {"inputs": {"name": "beta"}}) + + +@pytest.mark.parametrize( + ("overrides", "finalizes", "gates_passed"), + [ + ({}, True, "true"), + ({"needs__integration__result": "failure"}, True, "false"), + ({"needs__authenticated_test__result": "cancelled"}, True, "false"), + ({"needs__promote_entrypoint__result": "failure"}, True, "false"), + # Publication then a failure inside update_routing itself (#744 H2). + ({"needs__update_routing__result": "failure"}, True, "false"), + ( + { + "inputs__promote_entrypoint": False, + "needs__promote_entrypoint__result": "skipped", + }, + True, + "true", + ), + # Missing job outputs cannot prove nothing was published: runner loss + # can prevent reporting them even after upload and apply succeeded. + ( + { + "needs__update_routing__result": "failure", + "needs__update_routing__outputs": {}, + }, + True, + "false", + ), + ( + { + "needs__update_routing__result": "failure", + "needs__update_routing__outputs": {}, + "needs__integration__result": "skipped", + "needs__authenticated_test__result": "skipped", + "needs__promote_entrypoint__result": "skipped", + }, + True, + "false", + ), + # The restore does not depend on the plan's digests: even a plan that + # reported equal ones must be restored, because the restore is what + # resolves the release's publication record, which would otherwise + # refuse every later deployment. + ( + { + "needs__integration__result": "failure", + "needs__update_routing__outputs__after_sha256": "a" * 64, + }, + True, + "false", + ), + ( + { + "inputs__deploy_existing_stack": False, + "needs__integration__result": "failure", + }, + False, + "false", + ), + ( + {"needs__update_routing__result": "skipped"}, + False, + "false", + ), + # Stage12 alone never publishes, even when all its gates pass. + ( + { + "inputs__deploy_existing_stack": False, + "needs__update_routing__result": "skipped", + }, + False, + "false", + ), + # Re-run failed jobs carries a successful attempt-1 publication and + # its outputs. Finalization must use the run, never attempt-2 artifacts. + ( + {"github__run_attempt": "2"}, + True, + "true", + ), + ( + { + "github__run_attempt": "2", + "needs__authenticated_test__result": "failure", + }, + True, + "false", + ), + # A cancelled run after publication must still restore routing. + ( + { + "cancelled": True, + "needs__integration__result": "cancelled", + "needs__authenticated_test__result": "skipped", + "needs__promote_entrypoint__result": "skipped", + }, + True, + "false", + ), + ], + ids=[ + "all-pass", + "integration", + "authenticated", + "promotion", + "routing-job", + "no-promotion", + "missing-outputs", + "runner-lost-all-outputs", + "equal-digests", + "stack-disabled", + "routing-skipped", + "stage12-only", + "attempt-2-success", + "attempt-2-failed-gate", + "cancelled", + ], +) +def test_finalizer_always_resolves_a_publication_with_the_release_gate_outcome( + overrides, finalizes, gates_passed +): + finalizer = workflow_jobs(REUSABLE_DEPLOY_WORKFLOW)["finalize_routing"] + step = next( + step + for step in finalizer["steps"] + if "--finalize-publication" in step.get("run", "") + ) + context = routing_context(**overrides) + assert ( + bool(evaluate_github_expression(finalizer["if"], context, job_if=True)) + is finalizes + ) + assert str( + evaluate_github_expression(step["env"]["GATES_PASSED"], context) + ).lower() == (gates_passed) + + +@pytest.mark.parametrize("job", ["integration", "promote_entrypoint"]) +def test_a_failing_sentinel_fails_its_step(tmp_path, job): + """#744 review: `|| true` on the sentinel command must not pass a failure.""" + step = sentinel_check_step(workflow_jobs(REUSABLE_DEPLOY_WORKFLOW)[job]) + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + (bin_dir / "uv").write_text( + "#!/usr/bin/env bash\n" + 'if [[ "$*" == *"release_sentinel check"* ]]; then exit 9; fi\n' + "exit 0\n" + ) + (bin_dir / "uv").chmod(0o755) + script = step["run"].replace( + "${{ needs.deploy_entrypoint.outputs.candidate_url }}", "https://candidate" + ) + result = run_bash( + # GitHub runs `run` blocks with bash -e -o pipefail. + "--noprofile", + "--norc", + "-eo", + "pipefail", + "-c", + script, + check=False, + capture_output=True, + env={ + "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}", + "RUNNER_TEMP": str(tmp_path), + "STABLE_URL": "https://stable", + }, + ) + assert result.returncode == 9 + + +def test_stage12_only_promotion_runs_whatever_the_skipped_stack_jobs(): + """#743 review: the explicit status function must stay active.""" + promote = workflow_jobs(REUSABLE_DEPLOY_WORKFLOW)["promote_entrypoint"] + context = { + "inputs": {"promote_entrypoint": True, "deploy_existing_stack": False}, + "needs": { + "prepare": {"result": "success"}, + "deploy_entrypoint": {"result": "success"}, + "deploy_executor": {"result": "skipped"}, + "authenticated_test": {"result": "success"}, + "sentinel_expected": {"result": "skipped"}, + "update_routing": {"result": "skipped"}, + }, + } + assert evaluate_github_expression(promote["if"], context, job_if=True) + context["needs"]["authenticated_test"]["result"] = "failure" + assert not evaluate_github_expression(promote["if"], context, job_if=True) + context["needs"]["authenticated_test"]["result"] = "success" + context["inputs"]["promote_entrypoint"] = False + assert not evaluate_github_expression(promote["if"], context, job_if=True) + + +def test_smoke_rejects_a_latest_wrapper_that_lags_alone(tmp_path: Path): + """#743 review: the wrapper's latest is checked, not only the model's.""" + import copy + + versions = copy.deepcopy(VERSIONS_BODY) + versions["policyengine"]["latest"] = "6.2.6" + + result = run_identity_smoke(tmp_path, versions, **IDENTITY) + + assert result.returncode == 1 + assert "latest routes to policyengine 6.2.6 and policyengine-uk 2.126.8" in ( + result.stderr + ) diff --git a/projects/policyengine-simulation-entry/tests/test_deployment_assets_routing.py b/projects/policyengine-simulation-entry/tests/test_deployment_assets_routing.py new file mode 100644 index 000000000..c3263ce31 --- /dev/null +++ b/projects/policyengine-simulation-entry/tests/test_deployment_assets_routing.py @@ -0,0 +1,384 @@ +"""Tests for publishing and finalizing routing in the deployment workflows.""" + +from __future__ import annotations + +import json +import os +import re + +import pytest +from test_deployment_assets_support import ( + REPOSITORY_ROOT, + REUSABLE_DEPLOY_WORKFLOW, + ROLLBACK_WORKFLOW, + evaluate_github_expression, + job_section, + routing_context, + run_bash, + workflow_jobs, +) + + +def _run_routing_step_with_a_failing_command(tmp_path, step, failed_command, **env): + """Run the real body with Actions' shell and a failing CLI seam.""" + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + (bin_dir / "uv").write_text( + "#!/usr/bin/env bash\n" + 'printf "%s\\n" "$*" >> "$UV_CALLS"\n' + # Valid output ensures a masked plan/identity failure cannot be + # caught accidentally by a later grep rather than the failed CLI. + "printf 'before_sha256=before\\nafter_sha256=after\\nadvances_latest=true\\n'\n" + "printf 'policyengine_version=6.3.0\\napp_name=simulation-6.3.0\\n'\n" + "printf 'uk_model_version=2.126.8\\nuk_dataset_sha256=abc\\n'\n" + 'if [[ " $* " == *" ${FAILED_COMMAND} "* ]]; then exit 9; fi\n' + "exit 0\n" + ) + (bin_dir / "uv").chmod(0o755) + (bin_dir / "gh").write_text("#!/usr/bin/env bash\nprintf '1\\n'\n") + (bin_dir / "gh").chmod(0o755) + runner_temp = tmp_path / "runner-temp" + plan = runner_temp / "routing-plans" / "routing-plan-prod-1" / "plan.json" + plan.parent.mkdir(parents=True) + plan.write_text("{}") + result = run_bash( + "--noprofile", + "--norc", + "-eo", + "pipefail", + "-c", + step["run"], + check=False, + capture_output=True, + env={ + "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}", + "RUNNER_TEMP": str(runner_temp), + "GITHUB_OUTPUT": str(tmp_path / "github-output"), + "GITHUB_RUN_ID": "744001", + "GITHUB_RUN_ATTEMPT": "2", + "GITHUB_REPOSITORY": "PolicyEngine/policyengine-sim-api", + "UV_CALLS": str(tmp_path / "uv-calls"), + "FAILED_COMMAND": failed_command, + "MODAL_ENVIRONMENT": "main", + "PUBLICATION_RUN": "744001", + "RELEASE_ENVIRONMENT": "prod", + "GATES_PASSED": "true", + "APP_NAME": "simulation-6.3.0", + "POLICYENGINE_VERSION": "6.3.0", + "POLICYENGINE_US_VERSION": "1.0.0", + "POLICYENGINE_UK_VERSION": "2.126.8", + "FORCE_LATEST": "false", + "TARGET": "prod", + "DEPLOY_RUN_ID": "", + "EXPECTED_LATEST": "", + "RESOLVE_PENDING_PUBLICATION": "false", + **env, + }, + ) + assert result.returncode == 9, result.stderr + assert failed_command in (tmp_path / "uv-calls").read_text() + + +@pytest.mark.parametrize( + ("workflow", "job_name", "command", "env"), + [ + (REUSABLE_DEPLOY_WORKFLOW, "update_routing", "sync", {}), + (REUSABLE_DEPLOY_WORKFLOW, "finalize_routing", "sync", {}), + (ROLLBACK_WORKFLOW, "rollback", "sync", {}), + (REUSABLE_DEPLOY_WORKFLOW, "update_routing", "--plan-out", {}), + (REUSABLE_DEPLOY_WORKFLOW, "update_routing", "--record-plan", {}), + (REUSABLE_DEPLOY_WORKFLOW, "update_routing", "--apply-plan", {}), + (REUSABLE_DEPLOY_WORKFLOW, "finalize_routing", "--finalize-publication", {}), + ( + ROLLBACK_WORKFLOW, + "rollback", + "--restore-plan", + {"DEPLOY_RUN_ID": "744001"}, + ), + (ROLLBACK_WORKFLOW, "rollback", "--restore-latest", {}), + (ROLLBACK_WORKFLOW, "rollback", "--print-identity", {}), + ( + ROLLBACK_WORKFLOW, + "rollback", + "--print-pending-plan", + {"RESOLVE_PENDING_PUBLICATION": "true"}, + ), + ( + ROLLBACK_WORKFLOW, + "rollback", + "--finalize-publication", + {"RESOLVE_PENDING_PUBLICATION": "true"}, + ), + ], + ids=[ + "publication-setup", + "finalizer-setup", + "manual-rollback-setup", + "plan", + "record", + "apply", + "finalize", + "manual-restore-plans", + "manual-restore-latest", + "manual-read-identity", + "operator-inspect-pending", + "operator-resolve-pending", + ], +) +def test_a_failing_routing_command_fails_its_step_and_job( + tmp_path, workflow, job_name, command, env +): + job = workflow_jobs(workflow)[job_name] + assert "continue-on-error" not in job + assert job["defaults"]["run"]["shell"] == "bash" + step = next(step for step in job["steps"] if command in step.get("run", "")) + assert "continue-on-error" not in step + assert step.get("shell", "bash") == "bash" + _run_routing_step_with_a_failing_command(tmp_path, step, command, **env) + + +@pytest.mark.parametrize( + ("workflow", "job_name"), + [ + (REUSABLE_DEPLOY_WORKFLOW, "update_routing"), + (REUSABLE_DEPLOY_WORKFLOW, "finalize_routing"), + (REUSABLE_DEPLOY_WORKFLOW, "deployment_ready"), + (ROLLBACK_WORKFLOW, "rollback"), + ], +) +def test_routing_jobs_never_allow_command_failures(workflow, job_name): + job = workflow_jobs(workflow)[job_name] + assert "continue-on-error" not in job + assert job.get("defaults", {}).get("run", {}).get("shell") == "bash" + for step in job["steps"]: + assert "continue-on-error" not in step + assert step.get("shell", "bash") == "bash" + body = step.get("run", "") + assert not re.search(r"\|\|\s*true\b", body) + assert not re.search(r"\bset\s+\+e\b", body) + + +def test_each_publication_is_named_by_its_run_attempt_and_environment(): + routing = workflow_jobs(REUSABLE_DEPLOY_WORKFLOW)["update_routing"] + plan = next( + step + for step in routing["steps"] + if step.get("name") == "Plan the routing change" + ) + assert ( + '--publication-id "${GITHUB_RUN_ID}-' + '${GITHUB_RUN_ATTEMPT}-${RELEASE_ENVIRONMENT}"' in plan["run"] + ) + assert plan["env"]["RELEASE_ENVIRONMENT"] == "${{ inputs.release_environment }}" + + +def test_publication_keeps_the_plan_but_finalization_reads_the_journal(): + jobs = workflow_jobs(REUSABLE_DEPLOY_WORKFLOW) + routing = jobs["update_routing"] + names = [step.get("name") for step in routing["steps"]] + plan = names.index("Plan the routing change") + journal = names.index("Record the unfinished routing publication") + keep = names.index("Keep the plan for a guarded restore") + publish = names.index("Publish the planned routing") + assert "Record the plan artifact" not in names + # The plan is kept before anything is published. + assert plan < journal < keep < publish + assert "--record-plan" in routing["steps"][journal]["run"] + keep_step = routing["steps"][keep] + assert keep_step["uses"].startswith("actions/upload-artifact@") + assert keep_step["with"]["if-no-files-found"] == "error" + assert keep_step["with"]["name"] == ( + "routing-plan-${{ inputs.release_environment }}-${{ github.run_attempt }}" + ) + assert "--plan-out" in routing["steps"][plan]["run"] + assert "--apply-plan" in routing["steps"][publish]["run"] + assert routing["outputs"] == { + "before_sha256": "${{ steps.plan.outputs.before_sha256 }}", + "after_sha256": "${{ steps.plan.outputs.after_sha256 }}", + "advances_latest": "${{ steps.plan.outputs.advances_latest }}", + } + + assert "revert_routing" not in jobs + finalizer = jobs["finalize_routing"] + finalize = next( + step + for step in finalizer["steps"] + if "--finalize-publication" in step.get("run", "") + ) + assert not any( + step.get("uses", "").startswith("actions/download-artifact@") + for step in finalizer["steps"] + ) + assert '--publication-run "${PUBLICATION_RUN}"' in finalize["run"] + assert '--release-environment "${RELEASE_ENVIRONMENT}"' in finalize["run"] + assert '--gates-passed "${GATES_PASSED}"' in finalize["run"] + assert '--github-output "$GITHUB_OUTPUT"' in finalize["run"] + assert finalize["env"]["PUBLICATION_RUN"] == "${{ github.run_id }}" + assert finalize["env"]["RELEASE_ENVIRONMENT"] == ( + "${{ inputs.release_environment }}" + ) + assert "github.run_attempt" not in json.dumps(finalizer) + assert finalizer["outputs"]["outcome"] == ("${{ steps.finalize.outputs.outcome }}") + assert finalize["id"] == "finalize" + + cleanup = job_section( + REUSABLE_DEPLOY_WORKFLOW.read_text(encoding="utf-8"), + "cleanup_entrypoint_revisions", + ) + assert ( + "SIMULATION_ENTRYPOINT_ROLLBACK_REVISION: " + "${{ needs.deploy_entrypoint.outputs.previous_revision }}" + ) in cleanup + + +def test_deployment_ready_requires_the_journal_finalizer_to_complete(): + jobs = workflow_jobs(REUSABLE_DEPLOY_WORKFLOW) + ready = jobs["deployment_ready"] + assert { + "update_routing", + "integration", + "authenticated_test", + "promote_entrypoint", + "finalize_routing", + } <= set(ready["needs"]) + finalizer = jobs["finalize_routing"] + assert { + "update_routing", + "integration", + "authenticated_test", + "promote_entrypoint", + } <= set(finalizer["needs"]) + gate = next( + step for step in ready["steps"] if step.get("name") == "Check deployment result" + ) + assert "continue-on-error" not in gate + assert gate["env"]["EXISTING_STACK_REQUESTED"] == ( + "${{ inputs.deploy_existing_stack }}" + ) + assert gate["env"]["ROUTING_RESULT"] == "${{ needs.update_routing.result }}" + assert gate["env"]["FINALIZE_RESULT"] == "${{ needs.finalize_routing.result }}" + assert gate["env"]["FINALIZE_OUTCOME"] == ( + "${{ needs.finalize_routing.outputs.outcome }}" + ) + assert not any( + "--complete-plan" in step.get("run", "") + or step.get("uses", "").startswith("actions/download-artifact@") + for step in ready["steps"] + ) + + +@pytest.mark.parametrize( + "failed_gate", + [ + "ENTRYPOINT_RESULT", + "INTEGRATION_RESULT", + "AUTHENTICATED_TEST_RESULT", + "PROMOTION_RESULT", + "ROUTING_RESULT", + "FINALIZE_RESULT", + "FINALIZE_OUTCOME", + ], +) +def test_failed_release_gate_cannot_report_a_successful_release(tmp_path, failed_gate): + ready = workflow_jobs(REUSABLE_DEPLOY_WORKFLOW)["deployment_ready"] + gate = next( + step for step in ready["steps"] if step.get("name") == "Check deployment result" + ) + marker = tmp_path / "completed" + env = { + "PATH": os.environ["PATH"], + "PROMOTION_REQUESTED": "true", + "ENTRYPOINT_RESULT": "success", + "INTEGRATION_RESULT": "success", + "AUTHENTICATED_TEST_RESULT": "success", + "PROMOTION_RESULT": "success", + "EXISTING_STACK_REQUESTED": "true", + "ROUTING_RESULT": "success", + "FINALIZE_RESULT": "success", + "FINALIZE_OUTCOME": "completed", + } + env[failed_gate] = "failure" + # A later step cannot run after this gate under Actions' bash -e behavior. + result = run_bash( + "-eo", + "pipefail", + "-c", + gate["run"] + '\ntouch "$1"', + "gate", + str(marker), + check=False, + capture_output=True, + env=env, + cwd=REPOSITORY_ROOT, + ) + assert result.returncode != 0 + assert not marker.exists() + + +@pytest.mark.parametrize( + ("existing", "result", "outcome", "passes"), + [ + ("true", "success", "completed", True), + ("true", "success", "restored", False), + ("true", "success", "no-pending", False), + ("true", "success", "foreign-publication", False), + ("true", "failure", "completed", False), + ("true", "cancelled", "", False), + ("true", "skipped", "", False), + ("false", "skipped", "", True), + ], +) +def test_release_success_requires_completed_finalization_for_published_routing( + existing, result, outcome, passes +): + gate = next( + step + for step in workflow_jobs(REUSABLE_DEPLOY_WORKFLOW)["deployment_ready"]["steps"] + if step.get("name") == "Check deployment result" + ) + completed = run_bash( + "-eo", + "pipefail", + "-c", + gate["run"], + check=False, + capture_output=True, + cwd=REPOSITORY_ROOT, + env={ + "PATH": os.environ["PATH"], + "PROMOTION_REQUESTED": "false", + "ENTRYPOINT_RESULT": "success", + "INTEGRATION_RESULT": "success", + "AUTHENTICATED_TEST_RESULT": "success", + "PROMOTION_RESULT": "skipped", + "EXISTING_STACK_REQUESTED": existing, + "ROUTING_RESULT": "success" if existing == "true" else "skipped", + "FINALIZE_RESULT": result, + "FINALIZE_OUTCOME": outcome, + }, + ) + assert (completed.returncode == 0) is passes, completed.stderr + + +@pytest.mark.parametrize( + ("existing", "advances", "expected"), + [(True, "true", "true"), (True, "false", ""), (False, "true", ""), (True, "", "")], +) +def test_latest_is_expected_only_when_the_release_advances_it( + existing, advances, expected +): + jobs = workflow_jobs(REUSABLE_DEPLOY_WORKFLOW) + smoke_step = next( + step + for step in jobs["integration"]["steps"] + if "cloud-run-simulation-entry-smoke.sh" in step.get("run", "") + ) + context = routing_context( + inputs__deploy_existing_stack=existing, + needs__update_routing__outputs__advances_latest=advances, + ) + for environment in (smoke_step["env"], jobs["promote_entrypoint"]["env"]): + assert ( + evaluate_github_expression(environment["EXPECTED_LATEST"], context) + == expected + ) diff --git a/projects/policyengine-simulation-entry/tests/test_deployment_assets_support.py b/projects/policyengine-simulation-entry/tests/test_deployment_assets_support.py new file mode 100644 index 000000000..ae004c799 --- /dev/null +++ b/projects/policyengine-simulation-entry/tests/test_deployment_assets_support.py @@ -0,0 +1,411 @@ +"""Paths, shell and smoke fakes and a GitHub expression reader for deployment tests. + +It holds no tests. Its name keeps it among the test files the CI-script guard +searches for script names, and among the deployment test files the rules +registry discovers. +""" + +from __future__ import annotations + +import json +import os +import re +import subprocess +import textwrap +from pathlib import Path + +REPOSITORY_ROOT = Path(__file__).resolve().parents[3] +GITIGNORE = REPOSITORY_ROOT / ".gitignore" +TRAFFIC_SCRIPT = ( + REPOSITORY_ROOT + / ".github" + / "scripts" + / "set-cloud-run-simulation-entry-revision.sh" +) +SMOKE_SCRIPT = ( + REPOSITORY_ROOT / ".github" / "scripts" / "cloud-run-simulation-entry-smoke.sh" +) +STAGE12_VALIDATION_SCRIPT = ( + REPOSITORY_ROOT / ".github" / "scripts" / "stage12-validate-infrastructure.sh" +) +CLOUD_RUN_DEPLOY_SCRIPT = ( + REPOSITORY_ROOT / ".github" / "scripts" / "deploy-cloud-run-simulation-entry.sh" +) +DEPLOYMENT_READY_SCRIPT = ( + REPOSITORY_ROOT / ".github" / "scripts" / "simulation-deployment-ready.sh" +) +DEPLOY_WORKFLOW = REPOSITORY_ROOT / ".github" / "workflows" / "simulation-deploy.yml" +REUSABLE_DEPLOY_WORKFLOW = ( + REPOSITORY_ROOT / ".github" / "workflows" / "simulation-deploy.reusable.yml" +) +ROLLBACK_WORKFLOW = ( + REPOSITORY_ROOT / ".github" / "workflows" / "simulation-rollback.yml" +) +STAGE12_DEPLOY_WORKFLOW = ( + REPOSITORY_ROOT / ".github" / "workflows" / "stage12-deploy.yml" +) +PUBLISH_WORKFLOW = REPOSITORY_ROOT / ".github" / "workflows" / "publish-clients.yml" +AUTHENTICATED_TESTS = ( + REPOSITORY_ROOT + / "projects" + / "policyengine-simulation-entry" + / "authenticated_tests" +) +DOCKERIGNORE = ( + REPOSITORY_ROOT + / "projects" + / "policyengine-simulation-entry" + / "Dockerfile.dockerignore" +) +DOCKERFILE = ( + REPOSITORY_ROOT / "projects" / "policyengine-simulation-entry" / "Dockerfile" +) + + +def run_bash( + *arguments: str | Path, + check: bool, + capture_output: bool = False, + env: dict[str, str] | None = None, + cwd: Path | None = None, +) -> subprocess.CompletedProcess[str]: + """Run bash on a repository deployment script, with text output.""" + return subprocess.run( + ["bash", *arguments], + check=check, + capture_output=capture_output, + text=True, + env=env, + cwd=cwd, + ) + + +def write_fake_smoke_curl(tmp_path: Path, revisions: list[str]) -> tuple[Path, Path]: + state_file = tmp_path / "health-request-count" + revisions_file = tmp_path / "health-revisions" + revisions_file.write_text("\n".join(revisions) + "\n", encoding="utf-8") + fake_curl = tmp_path / "curl" + fake_curl.write_text( + textwrap.dedent("""\ + #!/usr/bin/env bash + set -euo pipefail + + headers_file="" + body_file="" + url="" + while [ "$#" -gt 0 ]; do + case "$1" in + --dump-header|--output|--request|--header|--data) + option="$1" + value="$2" + shift 2 + case "${option}" in + --dump-header) headers_file="${value}" ;; + --output) body_file="${value}" ;; + esac + ;; + --fail|--silent|--show-error) + shift + ;; + *) + url="$1" + shift + ;; + esac + done + + case "${url}" in + */health) + count=0 + if [ -f "${FAKE_CURL_STATE}" ]; then + count="$(cat "${FAKE_CURL_STATE}")" + fi + line_number=$((count + 1)) + revision="$(sed -n "${line_number}p" "${FAKE_CURL_REVISIONS}")" + printf '%s' "${line_number}" > "${FAKE_CURL_STATE}" + printf 'HTTP/1.1 200 OK\r\nX-PolicyEngine-Simulation-Revision: %s\r\n\r\n' \ + "${revision}" > "${headers_file}" + printf '{"status":"healthy"}' > "${body_file}" + ;; + */ready) + printf '{"status":"ready"}' + ;; + */versions) + versions='{}' + if [ -n "${FAKE_CURL_VERSIONS:-}" ]; then + versions="$(cat "${FAKE_CURL_VERSIONS}")" + fi + if [ -n "${body_file}" ]; then + printf '%s' "${versions}" > "${body_file}" + else + printf '%s' "${versions}" + fi + ;; + */ping) + printf '{"incremented":2}' + ;; + *) + printf 'Unexpected fake curl URL: %s\n' "${url}" >&2 + exit 2 + ;; + esac + """), # noqa: E501 # the fake curl writes its HTTP header block with one printf + encoding="utf-8", + ) + fake_curl.chmod(0o755) + return state_file, revisions_file + + +def smoke_test_env( + tmp_path: Path, + state_file: Path, + revisions_file: Path, +) -> dict[str, str]: + return { + **os.environ, + "PATH": f"{tmp_path}{os.pathsep}{os.environ['PATH']}", + "FAKE_CURL_STATE": str(state_file), + "FAKE_CURL_REVISIONS": str(revisions_file), + "SIMULATION_ENTRYPOINT_ROUTING_MAX_ATTEMPTS": "3", + "SIMULATION_ENTRYPOINT_ROUTING_RETRY_DELAY_SECONDS": "0", + } + + +def job_section(workflow: str, job_name: str) -> str: + match = re.search( + rf"^ {job_name}:\n(?P.*?)(?=^ [a-z0-9_-]+:\n|\Z)", + workflow, + flags=re.MULTILINE | re.DOTALL, + ) + assert match is not None, job_name + return match.group("body") + + +def sentinel_check_step(job: dict) -> dict: + steps = [ + step + for step in job["steps"] + if "src.modal.utils.release_sentinel check" in step.get("run", "") + ] + assert len(steps) == 1 + return steps[0] + + +# A small evaluator for the GitHub Actions expression subset these workflows +# use (&&, ||, !, ==, !=, parentheses, status functions, property access and +# string literals), so conditions are tested by what they decide, not by +# their text. It follows GitHub's rules where they matter here: a job ``if`` +# with no status function is implicitly ``success() && (...)``; ``==`` +# compares strings case-insensitively and converts mismatched types to +# numbers (true is 1, null is 0, a non-numeric string is NaN, so +# ``true == 'true'`` is false). +_TOKEN = re.compile(r"\s*(\$\{\{|\}\}|&&|\|\||==|!=|!|\(|\)|'[^']*'|[A-Za-z_][\w.-]*)") +_STATUS_FUNCTIONS = ("always", "success", "failure", "cancelled") + + +def _gh_number(value): + if value is None: + return 0.0 + if isinstance(value, bool): + return 1.0 if value else 0.0 + if isinstance(value, (int, float)): + return float(value) + text = str(value).strip() + if text == "": + return 0.0 + try: + return float(text) + except ValueError: + return float("nan") + + +def _gh_equal(left, right) -> bool: + if isinstance(left, str) and isinstance(right, str): + return left.lower() == right.lower() + if type(left) is type(right): + return left == right + return _gh_number(left) == _gh_number(right) + + +def _gh_tokens(expression: str) -> list[str]: + """Split an expression into tokens, dropping its ``${{`` and ``}}``.""" + tokens = [] + position = 0 + text = expression.strip() + while position < len(text): + match = _TOKEN.match(text, position) + assert match, text[position:] + tokens.append(match.group(1)) + position = match.end() + return [token for token in tokens if token not in ("${{", "}}")] + + +def _gh_calls_a_status_function(tokens: list[str]) -> bool: + return any( + token in _STATUS_FUNCTIONS and tokens[index + 1 : index + 2] == ["("] + for index, token in enumerate(tokens) + ) + + +def _gh_truthy(value) -> bool: + return value not in (None, False, "", 0) + + +class _GitHubExpression: + """A recursive-descent evaluation of one tokenized expression in a context. + + Each ``_parse_*`` method reads from a token index and returns the value + it read and the index after it, from the loosest operator (``||``) to the + tightest (``!``, parentheses and operands). + """ + + def __init__(self, tokens: list[str], context: dict) -> None: + self.tokens = tokens + self.context = context + self.results = [ + job.get("result") for job in (context.get("needs") or {}).values() + ] + + def evaluate(self): + value, index = self._parse_or(0) + assert index == len(self.tokens), self.tokens[index:] + return value + + def _next_is(self, index: int, operators: tuple[str, ...]) -> bool: + return index < len(self.tokens) and self.tokens[index] in operators + + def _status(self, name: str) -> bool: + cancelled = bool(self.context.get("cancelled")) + if name == "always": + return True + if name == "cancelled": + return cancelled + if name == "failure": + return "failure" in self.results + return not cancelled and all(result == "success" for result in self.results) + + def _lookup(self, path: str): + if path in ("true", "false"): + return path == "true" + value = self.context + for part in path.split("."): + value = value.get(part) if isinstance(value, dict) else None + return value + + def _parse_or(self, index: int): + left, index = self._parse_and(index) + while self._next_is(index, ("||",)): + right, index = self._parse_and(index + 1) + left = left if _gh_truthy(left) else right + return left, index + + def _parse_and(self, index: int): + left, index = self._parse_compare(index) + while self._next_is(index, ("&&",)): + right, index = self._parse_compare(index + 1) + left = right if _gh_truthy(left) else left + return left, index + + def _parse_compare(self, index: int): + left, index = self._parse_unary(index) + while self._next_is(index, ("==", "!=")): + operator = self.tokens[index] + right, index = self._parse_unary(index + 1) + equal = _gh_equal(left, right) + left = equal if operator == "==" else not equal + return left, index + + def _parse_unary(self, index: int): + symbol = self.tokens[index] + if symbol == "!": + value, index = self._parse_unary(index + 1) + return not _gh_truthy(value), index + if symbol == "(": + value, index = self._parse_or(index + 1) + assert self.tokens[index] == ")" + return value, index + 1 + if symbol.startswith("'"): + return symbol[1:-1], index + 1 + if self.tokens[index + 1 : index + 3] == ["(", ")"]: + assert symbol in _STATUS_FUNCTIONS, symbol + return self._status(symbol), index + 3 + return self._lookup(symbol), index + 1 + + +def evaluate_github_expression(expression: str, context: dict, *, job_if: bool = False): + tokens = _gh_tokens(expression) + if job_if and not _gh_calls_a_status_function(tokens): + tokens = ["success", "(", ")", "&&", "(", *tokens, ")"] + return _GitHubExpression(tokens, context).evaluate() + + +def routing_context(**overrides): + context = { + "inputs": {"deploy_existing_stack": True, "promote_entrypoint": True}, + "github": {"run_id": "744001", "run_attempt": "1"}, + "needs": { + "update_routing": { + "result": "success", + "outputs": { + "before_sha256": "a" * 64, + "after_sha256": "b" * 64, + "advances_latest": "true", + }, + }, + "integration": {"result": "success"}, + "authenticated_test": {"result": "success"}, + "promote_entrypoint": {"result": "success"}, + }, + } + for path, value in overrides.items(): + target = context + *parents, last = path.split("__") + for part in parents: + target = target[part] + target[last] = value + return context + + +# The gateway's real /versions response for two published releases (6.2.7 +# latest); test_versions_fixture.py in the gateway project keeps it current. +VERSIONS_BODY = json.loads( + (Path(__file__).parent / "fixtures" / "versions_body.json").read_text() +) + + +def run_identity_smoke(tmp_path: Path, versions: dict, **expected: str): + import json + + revision = "policyengine-simulation-entry-00002-new" + state_file, revisions_file = write_fake_smoke_curl(tmp_path, [revision]) + versions_file = tmp_path / "versions.json" + versions_file.write_text(json.dumps(versions), encoding="utf-8") + env = { + **smoke_test_env(tmp_path, state_file, revisions_file), + "FAKE_CURL_VERSIONS": str(versions_file), + **expected, + } + return run_bash( + SMOKE_SCRIPT, + "https://simulation.example.test", + revision, + check=False, + capture_output=True, + env=env, + ) + + +IDENTITY = { + "EXPECTED_IDENTITY": "required", + "EXPECTED_POLICYENGINE_VERSION": "6.2.7", + "EXPECTED_APP_NAME": "policyengine-simulation-py6-2-7", + "EXPECTED_UK_MODEL_VERSION": "2.126.8", + "EXPECTED_UK_DATASET_SHA256": "a" * 64, + "EXPECTED_LATEST": "true", +} + + +def workflow_jobs(path: Path) -> dict: + import yaml + + return yaml.safe_load(path.read_text(encoding="utf-8"))["jobs"] diff --git a/projects/policyengine-simulation-entry/tests/test_deployment_assets_traffic.py b/projects/policyengine-simulation-entry/tests/test_deployment_assets_traffic.py new file mode 100644 index 000000000..cda498290 --- /dev/null +++ b/projects/policyengine-simulation-entry/tests/test_deployment_assets_traffic.py @@ -0,0 +1,197 @@ +"""Tests for traffic changes, the container context, the lock and client publication.""" + +from __future__ import annotations + +import os +import textwrap + +import tomllib +from test_deployment_assets_support import ( + DOCKERIGNORE, + GITIGNORE, + PUBLISH_WORKFLOW, + REPOSITORY_ROOT, + TRAFFIC_SCRIPT, + run_bash, +) + + +def test_traffic_changes_are_exact_revision_validated_and_reversible(tmp_path): + script = TRAFFIC_SCRIPT.read_text(encoding="utf-8") + + assert "run revisions describe" in script + assert 'select(.type == "Ready" and .status == "True")' in script + assert '["serving.knative.dev/service"]' in script + assert 'run services update-traffic "${service}"' in script + assert '--to-revisions "${target_revision}=100"' in script + assert "expected_current_revision" in script + assert '--to-revisions "LATEST=100"' not in script + + state_file = tmp_path / "active-revision" + state_file.write_text("policyengine-simulation-entry-00001-old", encoding="utf-8") + fake_gcloud = tmp_path / "gcloud" + fake_gcloud.write_text( + textwrap.dedent("""\ + #!/usr/bin/env bash + set -euo pipefail + case "$1 $2 $3" in + "run services describe") + active="$(cat "${FAKE_GCLOUD_STATE}")" + printf '{"status":{"traffic":[{"revisionName":"%s","percent":100}]}}\\n' "${active}" + ;; + "run revisions describe") + revision="$4" + printf '{"metadata":{"labels":{"serving.knative.dev/service":"policyengine-simulation-entry"}},"status":{"conditions":[{"type":"Ready","status":"True"}]},"revision":"%s"}\\n' "${revision}" + ;; + "run services update-traffic") + target="" + while [ "$#" -gt 0 ]; do + if [ "$1" = "--to-revisions" ]; then + target="${2%=100}" + break + fi + shift + done + test -n "${target}" + printf '%s' "${target}" > "${FAKE_GCLOUD_STATE}" + ;; + *) + printf 'Unexpected fake gcloud command: %s\\n' "$*" >&2 + exit 2 + ;; + esac + """), # noqa: E501 # the fake gcloud prints each JSON response on one line, as gcloud does + encoding="utf-8", + ) + fake_gcloud.chmod(0o755) + + base_env = { + **os.environ, + "GCLOUD_BIN": str(fake_gcloud), + "FAKE_GCLOUD_STATE": str(state_file), + "SIMULATION_ENTRYPOINT_GCP_PROJECT_ID": "simulation-entry-test", + "SIMULATION_ENTRYPOINT_SERVICE": "policyengine-simulation-entry", + } + + run_bash( + TRAFFIC_SCRIPT, + check=True, + capture_output=True, + env={ + **base_env, + "SIMULATION_ENTRYPOINT_TARGET_REVISION": ( + "policyengine-simulation-entry-00002-new" + ), + "SIMULATION_ENTRYPOINT_EXPECTED_CURRENT_REVISION": ( + "policyengine-simulation-entry-00001-old" + ), + }, + ) + assert ( + state_file.read_text(encoding="utf-8") + == "policyengine-simulation-entry-00002-new" + ) + + run_bash( + TRAFFIC_SCRIPT, + check=True, + capture_output=True, + env={ + **base_env, + "SIMULATION_ENTRYPOINT_TARGET_REVISION": ( + "policyengine-simulation-entry-00001-old" + ), + "SIMULATION_ENTRYPOINT_EXPECTED_CURRENT_REVISION": ( + "policyengine-simulation-entry-00002-new" + ), + }, + ) + assert ( + state_file.read_text(encoding="utf-8") + == "policyengine-simulation-entry-00001-old" + ) + + +def test_traffic_change_refuses_an_intervening_promotion(tmp_path): + state_file = tmp_path / "active-revision" + state_file.write_text( + "policyengine-simulation-entry-00003-intervening", + encoding="utf-8", + ) + fake_gcloud = tmp_path / "gcloud" + fake_gcloud.write_text( + textwrap.dedent("""\ + #!/usr/bin/env bash + set -euo pipefail + active="$(cat "${FAKE_GCLOUD_STATE}")" + printf '{"status":{"traffic":[{"revisionName":"%s","percent":100}]}}\\n' "${active}" + """), # noqa: E501 # the fake gcloud prints its JSON response on one line, as gcloud does + encoding="utf-8", + ) + fake_gcloud.chmod(0o755) + + result = run_bash( + TRAFFIC_SCRIPT, + check=False, + capture_output=True, + env={ + **os.environ, + "GCLOUD_BIN": str(fake_gcloud), + "FAKE_GCLOUD_STATE": str(state_file), + "SIMULATION_ENTRYPOINT_GCP_PROJECT_ID": "simulation-entry-test", + "SIMULATION_ENTRYPOINT_SERVICE": "policyengine-simulation-entry", + "SIMULATION_ENTRYPOINT_TARGET_REVISION": ( + "policyengine-simulation-entry-00002-new" + ), + "SIMULATION_ENTRYPOINT_EXPECTED_CURRENT_REVISION": ( + "policyengine-simulation-entry-00001-old" + ), + }, + ) + + assert result.returncode == 2 + assert "Stable traffic changed after deployment" in result.stderr + + +def test_container_context_excludes_local_environments_and_unrelated_projects(): + ignore_rules = DOCKERIGNORE.read_text(encoding="utf-8") + gitignore_rules = GITIGNORE.read_text(encoding="utf-8") + + assert "gha-creds-*.json" in ignore_rules + assert "gha-creds-*.json" in gitignore_rules + assert "**/.venv" in ignore_rules + assert "projects/*" in ignore_rules + assert "!projects/policyengine-simulation-entry/**" in ignore_rules + assert "projects/policyengine-simulation-entry/authenticated_tests" in ignore_rules + assert "libs/*" in ignore_rules + assert ignore_rules.index("**/.venv") > ignore_rules.index( + "!projects/policyengine-simulation-entry/**" + ) + assert ignore_rules.index("**/.venv") > ignore_rules.index( + "!libs/policyengine-simulation-contract/**" + ) + + +def test_production_lock_contains_the_direct_stage12_database_runtime(): + lockfile = tomllib.loads( + ( + REPOSITORY_ROOT / "projects" / "policyengine-simulation-entry" / "uv.lock" + ).read_text(encoding="utf-8") + ) + resolved_packages = {package["name"] for package in lockfile["package"]} + + assert { + "httpx", + "modal", + "policyengine-stage12-persistence", + "psycopg", + "sqlalchemy", + } <= resolved_packages + for package in ("policyengine-fastapi", "sqlmodel"): + assert package not in resolved_packages + + +def test_client_publication_checks_out_the_deployed_commit(): + workflow = PUBLISH_WORKFLOW.read_text(encoding="utf-8") + + assert "github.event.workflow_run.head_sha" in workflow diff --git a/rules/baselines/python-file-length.txt b/rules/baselines/python-file-length.txt index 0f8e078d1..47a14f702 100644 --- a/rules/baselines/python-file-length.txt +++ b/rules/baselines/python-file-length.txt @@ -4,5 +4,3 @@ # Python file that was already over its line limit (800 lines, or 1,500 for # test code) when the rule was added. Remove an entry, and lower the rule's # baseline count, in the pull request that splits the file. -projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py -projects/policyengine-simulation-entry/tests/test_deployment_assets.py diff --git a/rules/registry.toml b/rules/registry.toml index 3f7a6679b..d1ddfcf08 100644 --- a/rules/registry.toml +++ b/rules/registry.toml @@ -59,14 +59,14 @@ guard_test_globs = [ "projects/*/tests/test_*boundar*.py", "libs/*/tests/test_*boundar*.py", "projects/*/tests/test_*schema_authority*.py", - "projects/policyengine-simulation-entry/tests/test_deployment_assets.py", + "projects/policyengine-simulation-entry/tests/test_deployment_assets*.py", "projects/policyengine-simulation-gateway/tests/test_openapi_golden.py", "projects/*/tests/test_runtime_evaluated_annotations.py", ] ignore_tests = [ - { node = "projects/policyengine-simulation-entry/tests/test_deployment_assets.py::test_the_expression_evaluator_follows_github_semantics", reason = "Checks the GitHub Actions expression evaluator that the workflow-condition tests use; it enforces no repository rule itself." }, - { node = "projects/policyengine-simulation-entry/tests/test_deployment_assets.py::test_the_expression_evaluator_returns_operands_and_job_statuses", reason = "Pins the operand values and job statuses the GitHub Actions expression evaluator returns; it enforces no repository rule itself." }, - { node = "projects/policyengine-simulation-entry/tests/test_deployment_assets.py::test_the_expression_evaluator_refuses_what_it_cannot_read", reason = "Pins that the GitHub Actions expression evaluator refuses expressions it cannot read; it enforces no repository rule itself." }, + { node = "projects/policyengine-simulation-entry/tests/test_deployment_assets_release_gates.py::test_the_expression_evaluator_follows_github_semantics", reason = "Checks the GitHub Actions expression evaluator that the workflow-condition tests use; it enforces no repository rule itself." }, + { node = "projects/policyengine-simulation-entry/tests/test_deployment_assets_release_gates.py::test_the_expression_evaluator_returns_operands_and_job_statuses", reason = "Pins the operand values and job statuses the GitHub Actions expression evaluator returns; it enforces no repository rule itself." }, + { node = "projects/policyengine-simulation-entry/tests/test_deployment_assets_release_gates.py::test_the_expression_evaluator_refuses_what_it_cannot_read", reason = "Pins that the GitHub Actions expression evaluator refuses expressions it cannot read; it enforces no repository rule itself." }, ] [agents] @@ -246,12 +246,12 @@ autofix = false kind = "pytest" project = "projects/policyengine-simulation-entry" nodes = [ - "tests/test_deployment_assets.py::test_main_deployment_automatically_deploys_stage12_in_both_environments", - "tests/test_deployment_assets.py::test_stage12_only_deployment_runs_candidate_tests_and_requested_promotion", - "tests/test_deployment_assets.py::test_stage12_only_deployment_cannot_redeploy_existing_modal_resources", - "tests/test_deployment_assets.py::test_stage12_only_deployment_skips_existing_stack_integration_suite", - "tests/test_deployment_assets.py::test_stage12_worker_deployment_can_request_a_google_oidc_token", - "tests/test_deployment_assets.py::test_stage12_only_promotion_runs_whatever_the_skipped_stack_jobs", + "tests/test_deployment_assets_promotion.py::test_main_deployment_automatically_deploys_stage12_in_both_environments", + "tests/test_deployment_assets_promotion.py::test_stage12_only_deployment_runs_candidate_tests_and_requested_promotion", + "tests/test_deployment_assets_promotion.py::test_stage12_only_deployment_cannot_redeploy_existing_modal_resources", + "tests/test_deployment_assets_promotion.py::test_stage12_only_deployment_skips_existing_stack_integration_suite", + "tests/test_deployment_assets_promotion.py::test_stage12_worker_deployment_can_request_a_google_oidc_token", + "tests/test_deployment_assets_release_gates.py::test_stage12_only_promotion_runs_whatever_the_skipped_stack_jobs", ] [[rule.ci]] workflow = ".github/workflows/pr.yml" @@ -270,7 +270,7 @@ autofix = false [rule.enforcer] kind = "pytest" project = "projects/policyengine-simulation-entry" -nodes = ["tests/test_deployment_assets.py::test_production_lock_contains_the_direct_stage12_database_runtime"] +nodes = ["tests/test_deployment_assets_traffic.py::test_production_lock_contains_the_direct_stage12_database_runtime"] [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" @@ -327,8 +327,8 @@ autofix = false kind = "pytest" project = "projects/policyengine-simulation-entry" nodes = [ - "tests/test_deployment_assets.py::test_cloud_run_deployment_does_not_interpolate_github_values_in_shell", - "tests/test_deployment_assets.py::test_cloud_run_deployment_escapes_environment_and_pins_secret_versions", + "tests/test_deployment_assets_cloud_run.py::test_cloud_run_deployment_does_not_interpolate_github_values_in_shell", + "tests/test_deployment_assets_cloud_run.py::test_cloud_run_deployment_escapes_environment_and_pins_secret_versions", ] [[rule.ci]] workflow = ".github/workflows/pr.yml" @@ -348,8 +348,8 @@ autofix = false kind = "pytest" project = "projects/policyengine-simulation-entry" nodes = [ - "tests/test_deployment_assets.py::test_cloud_run_revision_tags_are_valid_on_the_first_workflow_run", - "tests/test_deployment_assets.py::test_revision_tags_fit_existing_cloud_run_service_names", + "tests/test_deployment_assets_cloud_run.py::test_cloud_run_revision_tags_are_valid_on_the_first_workflow_run", + "tests/test_deployment_assets_cloud_run.py::test_revision_tags_fit_existing_cloud_run_service_names", ] [[rule.ci]] workflow = ".github/workflows/pr.yml" @@ -369,8 +369,8 @@ autofix = false kind = "pytest" project = "projects/policyengine-simulation-entry" nodes = [ - "tests/test_deployment_assets.py::test_full_stack_promotion_order_is_explicit", - "tests/test_deployment_assets.py::test_reusable_deployment_rejects_unexpected_skipped_phases", + "tests/test_deployment_assets_promotion.py::test_full_stack_promotion_order_is_explicit", + "tests/test_deployment_assets_promotion.py::test_reusable_deployment_rejects_unexpected_skipped_phases", ] [[rule.ci]] workflow = ".github/workflows/pr.yml" @@ -413,9 +413,9 @@ autofix = false kind = "pytest" project = "projects/policyengine-simulation-entry" nodes = [ - "tests/test_deployment_assets.py::test_cloud_run_deployment_defaults_permission_enforcement_to_log_only", - "tests/test_deployment_assets.py::test_cloud_run_deployment_rejects_invalid_permission_enforcement", - "tests/test_deployment_assets.py::test_permission_enforcement_reaches_candidate_and_authenticated_tests", + "tests/test_deployment_assets_cloud_run.py::test_cloud_run_deployment_defaults_permission_enforcement_to_log_only", + "tests/test_deployment_assets_cloud_run.py::test_cloud_run_deployment_rejects_invalid_permission_enforcement", + "tests/test_deployment_assets_cloud_run.py::test_permission_enforcement_reaches_candidate_and_authenticated_tests", ] [[rule.ci]] workflow = ".github/workflows/pr.yml" @@ -435,8 +435,8 @@ autofix = false kind = "pytest" project = "projects/policyengine-simulation-entry" nodes = [ - "tests/test_deployment_assets.py::test_traffic_changes_are_exact_revision_validated_and_reversible", - "tests/test_deployment_assets.py::test_traffic_change_refuses_an_intervening_promotion", + "tests/test_deployment_assets_traffic.py::test_traffic_changes_are_exact_revision_validated_and_reversible", + "tests/test_deployment_assets_traffic.py::test_traffic_change_refuses_an_intervening_promotion", ] [[rule.ci]] workflow = ".github/workflows/pr.yml" @@ -456,26 +456,26 @@ autofix = false kind = "pytest" project = "projects/policyengine-simulation-entry" nodes = [ - "tests/test_deployment_assets.py::test_deploy_gates_pass_the_release_identity_to_the_smoke", - "tests/test_deployment_assets.py::test_latest_is_expected_only_when_the_release_advances_it", - "tests/test_deployment_assets.py::test_smoke_without_expectations_only_requires_versions_json", - "tests/test_deployment_assets.py::test_smoke_accepts_the_gateway_identity_of_a_new_release", - "tests/test_deployment_assets.py::test_smoke_accepts_an_older_release_by_its_route", - "tests/test_deployment_assets.py::test_smoke_accepts_an_unrecorded_digest_only_when_none_is_served", - "tests/test_deployment_assets.py::test_smoke_requires_every_expectation_when_identity_is_required", - "tests/test_deployment_assets.py::test_smoke_refuses_an_incomplete_identity", - "tests/test_deployment_assets.py::test_smoke_refuses_malformed_expectations", - "tests/test_deployment_assets.py::test_smoke_rejects_identity_expectations_without_a_version", - "tests/test_deployment_assets.py::test_smoke_rejects_a_release_without_a_route", - "tests/test_deployment_assets.py::test_smoke_rejects_worker_routes_to_an_old_app", - "tests/test_deployment_assets.py::test_smoke_rejects_a_uk_model_routed_to_another_app", - "tests/test_deployment_assets.py::test_smoke_rejects_latest_routing_to_another_version", - "tests/test_deployment_assets.py::test_smoke_rejects_a_latest_wrapper_that_lags_alone", - "tests/test_deployment_assets.py::test_smoke_rejects_a_uk_latest_that_lags_the_wrapper", - "tests/test_deployment_assets.py::test_smoke_rejects_a_missing_bundle_identity", - "tests/test_deployment_assets.py::test_smoke_rejects_a_bundle_recorded_for_another_wrapper", - "tests/test_deployment_assets.py::test_smoke_rejects_another_uk_model", - "tests/test_deployment_assets.py::test_smoke_rejects_another_uk_dataset", + "tests/test_deployment_assets_identity_smoke.py::test_deploy_gates_pass_the_release_identity_to_the_smoke", + "tests/test_deployment_assets_routing.py::test_latest_is_expected_only_when_the_release_advances_it", + "tests/test_deployment_assets_identity_smoke.py::test_smoke_without_expectations_only_requires_versions_json", + "tests/test_deployment_assets_identity_smoke.py::test_smoke_accepts_the_gateway_identity_of_a_new_release", + "tests/test_deployment_assets_identity_smoke.py::test_smoke_accepts_an_older_release_by_its_route", + "tests/test_deployment_assets_identity_smoke.py::test_smoke_accepts_an_unrecorded_digest_only_when_none_is_served", + "tests/test_deployment_assets_identity_smoke.py::test_smoke_requires_every_expectation_when_identity_is_required", + "tests/test_deployment_assets_identity_smoke.py::test_smoke_refuses_an_incomplete_identity", + "tests/test_deployment_assets_identity_smoke.py::test_smoke_refuses_malformed_expectations", + "tests/test_deployment_assets_identity_smoke.py::test_smoke_rejects_identity_expectations_without_a_version", + "tests/test_deployment_assets_identity_smoke.py::test_smoke_rejects_a_release_without_a_route", + "tests/test_deployment_assets_identity_smoke.py::test_smoke_rejects_worker_routes_to_an_old_app", + "tests/test_deployment_assets_identity_smoke.py::test_smoke_rejects_a_uk_model_routed_to_another_app", + "tests/test_deployment_assets_identity_smoke.py::test_smoke_rejects_latest_routing_to_another_version", + "tests/test_deployment_assets_release_gates.py::test_smoke_rejects_a_latest_wrapper_that_lags_alone", + "tests/test_deployment_assets_identity_smoke.py::test_smoke_rejects_a_uk_latest_that_lags_the_wrapper", + "tests/test_deployment_assets_identity_smoke.py::test_smoke_rejects_a_missing_bundle_identity", + "tests/test_deployment_assets_identity_smoke.py::test_smoke_rejects_a_bundle_recorded_for_another_wrapper", + "tests/test_deployment_assets_identity_smoke.py::test_smoke_rejects_another_uk_model", + "tests/test_deployment_assets_identity_smoke.py::test_smoke_rejects_another_uk_dataset", ] [[rule.ci]] workflow = ".github/workflows/pr.yml" @@ -495,8 +495,8 @@ autofix = false kind = "pytest" project = "projects/policyengine-simulation-entry" nodes = [ - "tests/test_deployment_assets.py::test_deployed_stack_must_reproduce_the_numerical_sentinel", - "tests/test_deployment_assets.py::test_a_failing_sentinel_fails_its_step", + "tests/test_deployment_assets_promotion.py::test_deployed_stack_must_reproduce_the_numerical_sentinel", + "tests/test_deployment_assets_release_gates.py::test_a_failing_sentinel_fails_its_step", ] [[rule.ci]] workflow = ".github/workflows/pr.yml" @@ -516,14 +516,14 @@ autofix = false kind = "pytest" project = "projects/policyengine-simulation-entry" nodes = [ - "tests/test_deployment_assets.py::test_each_publication_is_named_by_its_run_attempt_and_environment", - "tests/test_deployment_assets.py::test_publication_keeps_the_plan_but_finalization_reads_the_journal", - "tests/test_deployment_assets.py::test_finalizer_always_resolves_a_publication_with_the_release_gate_outcome", - "tests/test_deployment_assets.py::test_deployment_ready_requires_the_journal_finalizer_to_complete", - "tests/test_deployment_assets.py::test_release_success_requires_completed_finalization_for_published_routing", - "tests/test_deployment_assets.py::test_failed_release_gate_cannot_report_a_successful_release", - "tests/test_deployment_assets.py::test_a_failing_routing_command_fails_its_step_and_job", - "tests/test_deployment_assets.py::test_routing_jobs_never_allow_command_failures", + "tests/test_deployment_assets_routing.py::test_each_publication_is_named_by_its_run_attempt_and_environment", + "tests/test_deployment_assets_routing.py::test_publication_keeps_the_plan_but_finalization_reads_the_journal", + "tests/test_deployment_assets_release_gates.py::test_finalizer_always_resolves_a_publication_with_the_release_gate_outcome", + "tests/test_deployment_assets_routing.py::test_deployment_ready_requires_the_journal_finalizer_to_complete", + "tests/test_deployment_assets_routing.py::test_release_success_requires_completed_finalization_for_published_routing", + "tests/test_deployment_assets_routing.py::test_failed_release_gate_cannot_report_a_successful_release", + "tests/test_deployment_assets_routing.py::test_a_failing_routing_command_fails_its_step_and_job", + "tests/test_deployment_assets_routing.py::test_routing_jobs_never_allow_command_failures", ] [[rule.ci]] workflow = ".github/workflows/pr.yml" @@ -543,8 +543,8 @@ autofix = false kind = "pytest" project = "projects/policyengine-simulation-entry" nodes = [ - "tests/test_deployment_assets.py::test_container_context_excludes_local_environments_and_unrelated_projects", - "tests/test_deployment_assets.py::test_entrypoint_container_uses_requested_target_platform", + "tests/test_deployment_assets_traffic.py::test_container_context_excludes_local_environments_and_unrelated_projects", + "tests/test_deployment_assets_cloud_run.py::test_entrypoint_container_uses_requested_target_platform", ] [[rule.ci]] workflow = ".github/workflows/pr.yml" @@ -655,7 +655,7 @@ autofix = false [rule.enforcer] kind = "pytest" project = "projects/policyengine-simulation-entry" -nodes = ["tests/test_deployment_assets.py::test_complete_integration_suite_is_configured_for_beta_only"] +nodes = ["tests/test_deployment_assets_promotion.py::test_complete_integration_suite_is_configured_for_beta_only"] [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" @@ -673,7 +673,7 @@ autofix = false [rule.enforcer] kind = "pytest" project = "projects/policyengine-simulation-entry" -nodes = ["tests/test_deployment_assets.py::test_client_publication_checks_out_the_deployed_commit"] +nodes = ["tests/test_deployment_assets_traffic.py::test_client_publication_checks_out_the_deployed_commit"] [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" @@ -1190,7 +1190,7 @@ job = "quality-guards" invocation = "run_quality_guards.py" [rule.baseline] file = "rules/baselines/python-file-length.txt" -count = 2 +count = 0 [rule.proof] bad_fixture = "rules/fixtures/sim.quality.python-file-length/bad_files.toml" good_fixture = "rules/fixtures/sim.quality.python-file-length/good_files.toml"