Run and store v2 economy reports in the Simulation Entrypoint

.github/scripts/deploy-cloud-run-simulation-safety-check.sh

@@ -6,8 +6,9 @@
 # recording pass per execution. It is updated only from a revision that was
 # promoted (or restored by a rollback), so a rejected candidate never records
 # anything. It copies the revision's image digest, service account, Modal
-# token and database URL (at the secret versions the revision pins) and its
-# plain telemetry settings; nothing else. A revision deployed before the
+# token and database URL (at the secret versions the revision pins), its
+# plain telemetry settings, and the Modal environment it starts the economy
+# calls left waiting in; nothing else. A revision deployed before the
 # safety check existed has no job code, so its schedule is paused instead.
 #
 # With the argument `pause`, only pause the schedule: a workflow does this
@@ -33,7 +34,7 @@ pause_schedule() {
 # policyengine_simulation_entry.run_recording.
 SCHEDULE="*/5 * * * *"
 SECRETS=(SIMULATION_DATABASE_URL MODAL_TOKEN_ID MODAL_TOKEN_SECRET)
-PLAIN_VARIABLES_FILTER='^(APP_ENVIRONMENT|CLOUD_RUN_REGION|OBSERVABILITY_[A-Z_]+|OTEL_[A-Z_]+|POLICYENGINE_OTEL_GOOGLE_AUDIENCE)$'
+PLAIN_VARIABLES_FILTER='^(APP_ENVIRONMENT|CLOUD_RUN_REGION|SIMULATION_CALL_REPORT_MODAL_ENVIRONMENT|OBSERVABILITY_[A-Z_]+|OTEL_[A-Z_]+|POLICYENGINE_OTEL_GOOGLE_AUDIENCE)$'
 
 for variable_name in PROJECT_ID REGION SERVICE; do
   [[ -n "${!variable_name:-}" ]] || fail "${variable_name} is required"

.github/scripts/public-api-schema-reference.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+# Build the Public API's tables with its own v2 migration chain, for the
+# comparison with the persistence tests' copies of them (rule
+# sim.persistence.public-api-copies-match).
+#
+# Usage: public-api-schema-reference.sh <Public API checkout>
+#
+# PUBLIC_API_SCHEMA_DATABASE_URL names the empty database to migrate, as
+# postgresql://user:password@host:port/database. The Public API's disposable
+# migration mode accepts only the local policyengine_v2_alembic_test
+# database, with explicit credentials. The script prints the Public API
+# commit it migrated and, on GitHub Actions, passes it to the later steps as
+# PUBLIC_API_SCHEMA_COMMIT, so a failed comparison names it.
+set -euo pipefail
+
+checkout="${1:?usage: public-api-schema-reference.sh <Public API checkout>}"
+database_url="${PUBLIC_API_SCHEMA_DATABASE_URL:?PUBLIC_API_SCHEMA_DATABASE_URL must name the database to migrate}"
+
+cd "$checkout"
+commit="$(git rev-parse HEAD)"
+echo "Public API commit: $commit"
+if [ -n "${GITHUB_ENV:-}" ]; then
+  echo "PUBLIC_API_SCHEMA_COMMIT=$commit" >> "$GITHUB_ENV"
+fi
+# The Public API installs its lock on Python 3.12, as its own CI does.
+export UV_PYTHON=3.12
+uv sync --frozen --extra dev
+V2_MIGRATION_DATABASE_URL="postgresql+psycopg://${database_url#*://}" \
+  V2_ALEMBIC_DISPOSABLE_TEST=1 \
+  uv run --frozen --extra dev alembic -c alembic-v2.ini upgrade head

.github/workflows/pr.yml

@@ -16,7 +16,7 @@ jobs:
     # every job passed. See docs/engineering/skills/github-prs.md.
     name: CI complete
     if: ${{ always() }}
-    needs: [test, simulation-database, lint, pyright, quality-guards, docker-build, test-integration]
+    needs: [test, simulation-database, public-api-schema, lint, pyright, quality-guards, docker-build, test-integration]
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -124,6 +124,76 @@ jobs:
       working-directory: libs/policyengine-simulation-persistence
       run: uv run --frozen pytest tests/ -v -m postgres
 
+  public-api-schema:
+    name: Public API schema
+    # Compares the persistence tests' copies of the Public API's tables with
+    # the tables the Public API's own v2 migration chain builds, at the Public
+    # API's default branch, so a change there to a column sim reads fails
+    # here (rule sim.persistence.public-api-copies-match). The Public API's
+    # repository is public, so the checkout needs no secret. Full CI runs
+    # only on the lowest unmerged pull request in a stack.
+    if: ${{ !github.event.pull_request.stack || github.event.pull_request.stack.base.ref == github.event.pull_request.base.ref }}
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    services:
+      postgres:
+        image: postgres:17
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: simulation_ci
+          # The Public API's disposable migration mode accepts only this
+          # database; the server's postgres database holds sim's copies.
+          POSTGRES_DB: policyengine_v2_alembic_test
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd="pg_isready -U postgres"
+          --health-interval=5s
+          --health-timeout=5s
+          --health-retries=20
+    env:
+      SIMULATION_TEST_DATABASE_URL: postgresql://postgres:simulation_ci@127.0.0.1:5432/postgres
+      PUBLIC_API_SCHEMA_DATABASE_URL: postgresql://postgres:simulation_ci@127.0.0.1:5432/policyengine_v2_alembic_test
+      # A missing server or Public API database fails the comparison instead
+      # of skipping it.
+      SIMULATION_REQUIRE_DATABASE_TESTS: "1"
+      SIMULATION_REQUIRE_PUBLIC_API_SCHEMA: "1"
+
+    steps:
+    - uses: actions/checkout@v6
+
+    - name: Check out the Public API's default branch
+      uses: actions/checkout@v6
+      with:
+        repository: PolicyEngine/policyengine-api
+        path: public-api
+        persist-credentials: false
+
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        # 3.12 for the Public API's lock, as its own CI uses; 3.13 for sim's.
+        python-version: |
+          3.12
+          3.13
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v8.1.0
+      with:
+        enable-cache: true
+
+    - name: Build the Public API's tables from its v2 migration chain
+      run: .github/scripts/public-api-schema-reference.sh public-api
+
+    - name: Install dependencies
+      working-directory: libs/policyengine-simulation-persistence
+      run: uv sync --extra test --frozen
+
+    - name: Compare the test copies with the Public API's tables
+      working-directory: libs/policyengine-simulation-persistence
+      run: uv run --frozen pytest tests/ -v -m public_api_schema
+
   lint:
     name: Lint
     # Full CI runs only on the lowest unmerged pull request in a stack.

.github/workflows/simulation-deploy.reusable.yml

@@ -1027,6 +1027,50 @@ jobs:
         if: ${{ failure() && steps.safety_check_restore.outcome == 'failure' }}
         run: .github/scripts/deploy-cloud-run-simulation-safety-check.sh pause
 
+  economy_staging_check:
+    name: Run a small v2 economy report on staging
+    # After the staging promotion, the stable URL serves the revision this
+    # deployment tested: submit a small report there, run it again with
+    # fresh runs if the submission reused runs an earlier deployment
+    # finished, and read it back once its comparison is recorded. The v2
+    # economy path serves no user yet, so a failure is reported without
+    # holding back the production deployment.
+    if: >-
+      ${{
+        !cancelled() &&
+        inputs.deployment_environment == 'staging' &&
+        needs.promote_entrypoint.result == 'success'
+      }}
+    needs: [prepare, deploy_entrypoint, promote_entrypoint]
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    timeout-minutes: 30
+    environment: ${{ inputs.release_environment }}
+    env:
+      SIMULATION_ENTRYPOINT_TEST_BASE_URL: ${{ needs.deploy_entrypoint.outputs.stable_url }}
+      SIMULATION_ENTRYPOINT_TEST_AUTH_ISSUER: ${{ secrets.SIMULATION_ENTRYPOINT_AUTH_ISSUER }}
+      SIMULATION_ENTRYPOINT_TEST_AUTH_AUDIENCE: ${{ secrets.SIMULATION_ENTRYPOINT_AUTH_AUDIENCE }}
+      SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_ID: ${{ secrets.SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_ID }}
+      SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_SECRET: ${{ secrets.SIMULATION_ENTRYPOINT_TEST_AUTH_CLIENT_SECRET }}
+      SIMULATION_STAGING_ECONOMY_POLICYENGINE_VERSION: ${{ needs.prepare.outputs.policyengine_version }}
+      SIMULATION_STAGING_ECONOMY_CHECK: "1"
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.13"
+
+      - uses: astral-sh/setup-uv@v8.1.0
+
+      - name: Install staging check dependencies
+        working-directory: projects/policyengine-simulation-entry
+        run: uv sync --extra test --frozen
+
+      - name: Run a small economy report on staging and read it back
+        working-directory: projects/policyengine-simulation-entry
+        run: uv run --frozen pytest authenticated_tests/test_staging_economy_report.py -v
+
   finalize_routing:
     name: Finalize the Modal routing publication
     # Finish the durable journal after every gate, including failed uploads

.github/workflows/simulation-deploy.yml

@@ -40,7 +40,10 @@ jobs:
       entrypoint_revision_prefix: s-
       entrypoint_min_instances: 0
       entrypoint_max_instances: 2
-      promote_entrypoint: false
+      # Staging's stable URL must serve the revision this deployment tested,
+      # because staging callers such as the Public API's staging service use
+      # that URL. Rule: sim.deploy.staging-promotes-tested-revision.
+      promote_entrypoint: true
       run_full_integration: true
       deploy_existing_stack: true
       deploy_stage12_v2: true

.github/workflows/stage12-deploy.yml

@@ -53,7 +53,9 @@ jobs:
       entrypoint_revision_prefix: s-
       entrypoint_min_instances: 0
       entrypoint_max_instances: 2
-      promote_entrypoint: false
+      # Like the main deployment, promote the tested staging revision.
+      # Rule: sim.deploy.staging-promotes-tested-revision.
+      promote_entrypoint: true
       # This workflow deploys and validates only the independent Stage 12 path.
       # The main API deployment continues to run the existing-stack suite.
       run_full_integration: false

AGENTS.md

@@ -124,11 +124,11 @@ To add or change a rule, read `docs/engineering/skills/rules-registry.md`.
 
 ### Simulation persistence
 
-- Checked: `sim.persistence.executor-independent`, `sim.persistence.migrate-before-entrypoint`, `sim.persistence.migration-cycle`, `sim.persistence.no-modal-database-credentials`, `sim.persistence.one-migration-chain`
+- Checked: `sim.persistence.executor-independent`, `sim.persistence.migrate-before-entrypoint`, `sim.persistence.migration-cycle`, `sim.persistence.no-modal-database-credentials`, `sim.persistence.one-migration-chain`, `sim.persistence.public-api-copies-match`
 
 ### Deployment pipeline
 
-- Checked: `sim.deploy.container-context-minimal`, `sim.deploy.docker-images-build`, `sim.deploy.gcloud-without-terraform`, `sim.deploy.modal-image-smoke`, `sim.deploy.numerical-sentinel-gate`, `sim.deploy.permission-enforcement-wired`, `sim.deploy.promotion-order-explicit`, `sim.deploy.release-identity-smoke`, `sim.deploy.revision-tags-valid`, `sim.deploy.routing-publication-journaled`, `sim.deploy.shell-scripts-valid`, `sim.deploy.smoke-tests-strict`, `sim.deploy.traffic-changes-exact`, `sim.deploy.workflow-shell-safety`
+- Checked: `sim.deploy.container-context-minimal`, `sim.deploy.docker-images-build`, `sim.deploy.gcloud-without-terraform`, `sim.deploy.modal-image-smoke`, `sim.deploy.numerical-sentinel-gate`, `sim.deploy.permission-enforcement-wired`, `sim.deploy.promotion-order-explicit`, `sim.deploy.release-identity-smoke`, `sim.deploy.revision-tags-valid`, `sim.deploy.routing-publication-journaled`, `sim.deploy.shell-scripts-valid`, `sim.deploy.smoke-tests-strict`, `sim.deploy.staging-promotes-tested-revision`, `sim.deploy.traffic-changes-exact`, `sim.deploy.workflow-shell-safety`
 - `sim.deploy.image-smoke-covers-every-image-lib`: The image smoke path filter includes every library mounted into a Modal image, so a change to any of them runs the smoke. (doc-only, not checked: follow it yourself)
 
 ### API contracts and integration

README.md

@@ -91,13 +91,20 @@ time:
    three parallel service jobs;
 3. publish routing only after all three service jobs pass;
 4. run generated-client and live authentication tests through the complete
-   three-service request path; and
-5. begin the `prod` deployment only after every `beta` deployment and
-   qualification job succeeds;
-6. assign stable production traffic to the exact tested entrypoint revision
-   after the complete production suite passes; and
-7. verify the stable endpoint, restoring its previous revision if immediate
-   post-promotion checks fail.
+   three-service request path;
+5. assign the environment's stable traffic to the exact tested entrypoint
+   revision after that environment's tests pass, in `beta` as in `prod`;
+6. verify the stable endpoint, restoring its previous revision if immediate
+   post-promotion checks fail; and
+7. begin the `prod` deployment only after every `beta` deployment,
+   qualification, promotion and revision clean-up job succeeds, so a failed
+   staging clean-up also stops the production deployment.
+
+The staging service's stable URL therefore serves the revision that passed the
+staging tests, so staging callers of that URL, such as the Public API's staging
+service, reach the build the deployment tested. After each successful
+promotion, the deployment deletes the service's older revisions, keeping only
+the promoted revision and the one it replaced, for rollback.
 
 A successful push-triggered full-stack deployment publishes the API client from
 the exact deployed commit.

changelog_entry.yaml

@@ -27,6 +27,7 @@
       - Split the routing publisher (src/modal/utils/update_version_registry.py, 1,584 lines) into a src/modal/routing_publisher package of modules for the bundle manifest, the routing state, routing plans and their publication record, the check that only serialized workflows write routing, Modal Dict access, plan publication, the legacy seed and the command-line options, and its 2,728-line test file into a tests/version_registry directory of nine test files with shared fixtures; update_version_registry.py keeps the command line, still runs with python -m src.modal.utils.update_version_registry and re-exports every public name; behaviour and output are unchanged, and the private helpers the new modules share are now public
       - Removed the UK constituency and local-authority weight-matrix region fallback (_build_uk_weight_replacement_region, _require_certified_uk_weight_matrix and its Stage 12 call); each deployment rebuilds only the app for the current policyengine.py version, now 6.10.0, which routes those regions to the local-area dataset, and apps deployed for older versions are not rebuilt and keep their own code; on a bundle that does not route them, an area region is now an unsupported region
       - Made UK-wide and nation-level runs on a policyengine.py bundle that routes UK area regions to a local-area release dataset (6.10.0) load only the national dataset and return null for constituency_impact and local_authority_impact, whose results come from a breakdown=local_areas run instead; on a bundle that does not route them, those runs still compute both from the national dataset, and Stage 12 leaves the same breakdowns out of the same reports
+      - Promoted each staging deployment's tested Simulation Entrypoint revision to the staging service's stable traffic, as production deployments already do, in both the main deployment's beta job and the Stage 12 comparison path's staging deployment; the staging stable URL now serves the build that passed the staging tests, promotion runs the same stable-URL checks as production's and restores the previous revision when they fail, and the existing clean-up then deletes the staging service's older revisions except the promoted one and the one it replaced; the new rule sim.deploy.staging-promotes-tested-revision requires it
       - Enabled every ruff rule (select ALL), ignoring only COM812 and ISC001, which conflict with ruff format, and the D203 and D213 halves of the mutually exclusive docstring pairs; complex functions, long parameter lists and magic numbers were split into smaller helpers and grouped parameters, each behind new characterization tests, across the contract and persistence validation, entry settings, app factory and Stage 12 adapter, gateway route resolution, Stage 12 coordination, workers, parity and qualification, the precompute and national reduce, the routing publisher and its command line, the release sentinel, the package update helper and the shared rules tool; Modal function, FastAPI route, Pydantic validator and command-line signatures and every OpenAPI document are unchanged; the contract library's batch-state builder and SPM selection resolver take their route values as one grouped argument, the simulation observability initializer drops a service_version parameter no caller passed, and no PLR0913 suppression is needed
       - Enabled ruff's argument, idiom and pytest-style rules (ARG, ASYNC, B, C4, FBT, PD, PERF, PGH, PLW, PT, PTH, RUF, SIM); unused arguments are removed, underscored or explained, FastAPI auth dependencies use Annotated, re-raised errors keep their cause, and boolean options of in-repo helpers are keyword-only; OpenAPI documents are unchanged, and the shared rules tool's unittest tests are exempt from pytest-style rules; tests take the unused-argument rules (ARG001, ARG002, ARG005) as file-level exceptions, since their fakes and stubs mirror real signatures; best-effort telemetry and cleanup calls that swallow any error use contextlib.suppress, with the reason as a comment
       - Enabled ruff's structure and import rules (INP, PLC, SLF, N, A, FAST, E); long lines are wrapped, deferred imports that Modal images, test seams, import cycles or heavy dependencies need carry a reason, script directories are declared to ruff as namespace packages, four internal exceptions gained an Error suffix, and published field names, OpenAPI documents and logged error types are unchanged; tests are also exempt from SLF001, INP001 and PLC0415; heavy or optional packages that load only inside functions are listed in ruff's banned-module-level-imports, which keeps them out of module-level imports (TID253) and lets their deferred imports go without a suppression, and the Modal utility programs take INP001 file-wide