Record simulation runs and report runs, and their outcomes when Modal reports a call ended

.github/scripts/deploy-cloud-run-simulation-entry.sh

@@ -100,6 +100,9 @@ required_variables=(
   OLD_GATEWAY_AUTH_CLIENT_ID_VALUE
   OLD_GATEWAY_AUTH_CLIENT_SECRET_SECRET_NAME
   SIMULATION_DATABASE_URL_SECRET_NAME
+  MODAL_TOKEN_ID_SECRET_NAME
+  MODAL_TOKEN_SECRET_SECRET_NAME
+  MODAL_WORKSPACE_ID_VALUE
   OBSERVABILITY_SERVICE_NAMESPACE
   OBSERVABILITY_TRACE_PROJECT_ID
   OTEL_EXPORTER_OTLP_ENDPOINT
@@ -126,6 +129,10 @@ esac
   || fail "TAG is invalid"
 [[ "${ENTRYPOINT_RUNTIME_SERVICE_ACCOUNT}" =~ ^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.gserviceaccount\.com$ ]] \
   || fail "ENTRYPOINT_RUNTIME_SERVICE_ACCOUNT is invalid"
+# The Modal workspace whose call-report function may report that calls
+# ended (the entrypoint also checks the environment, MODAL_ENVIRONMENT).
+[[ "${MODAL_WORKSPACE_ID_VALUE}" =~ ^ac-[A-Za-z0-9]+$ ]] \
+  || fail "MODAL_WORKSPACE_ID_VALUE must be a Modal workspace ID"
 [[ "${ENTRYPOINT_MIN_INSTANCES}" =~ ^[0-9]+$ ]] \
   || fail "ENTRYPOINT_MIN_INSTANCES must be a non-negative integer"
 [[ "${ENTRYPOINT_MAX_INSTANCES}" =~ ^[0-9]+$ ]] \
@@ -143,13 +150,16 @@ add_runtime_secret OLD_GATEWAY_AUTH_CLIENT_SECRET \
 # The runtime role's URL for the entrypoint's own simulation schema. The
 # migration job, which uses a separate migrator URL, runs before this deploy.
 add_runtime_secret SIMULATION_DATABASE_URL "${SIMULATION_DATABASE_URL_SECRET_NAME}"
+# sim's own Modal token, on every deploy: the entrypoint starts runs' Modal
+# calls and asks Modal about a call when Modal reports that it ended. Until
+# the Stage 12 comparison path is removed, its Modal calls use the same token.
+add_runtime_secret MODAL_TOKEN_ID "${MODAL_TOKEN_ID_SECRET_NAME}"
+add_runtime_secret MODAL_TOKEN_SECRET "${MODAL_TOKEN_SECRET_SECRET_NAME}"
 
 if [[ "${DEPLOY_STAGE12_V2}" == "true" ]]; then
   stage12_variables=(
     STAGE12_ENABLED_VALUE
     STAGE12_ARTIFACT_BUCKET_VALUE
-    MODAL_TOKEN_ID_SECRET_NAME
-    MODAL_TOKEN_SECRET_SECRET_NAME
     STAGE12_DATABASE_URL_SECRET_NAME
   )
   for variable_name in "${stage12_variables[@]}"; do
@@ -161,8 +171,6 @@ if [[ "${DEPLOY_STAGE12_V2}" == "true" ]]; then
     || fail "STAGE12_ARTIFACT_BUCKET_VALUE is invalid"
   validate_secret_reference "${STAGE12_DATABASE_URL_SECRET_NAME}"
 
-  add_stage12_runtime_secret MODAL_TOKEN_ID "${MODAL_TOKEN_ID_SECRET_NAME}"
-  add_stage12_runtime_secret MODAL_TOKEN_SECRET "${MODAL_TOKEN_SECRET_SECRET_NAME}"
   add_stage12_runtime_secret STAGE12_DATABASE_URL "${STAGE12_DATABASE_URL_SECRET_NAME}"
 fi
 
@@ -188,6 +196,8 @@ jq -n '
     OTEL_METRICS_EXPORTER: "otlp",
     OTEL_TRACES_SAMPLER_ARG: "1.0",
     POLICYENGINE_OTEL_GOOGLE_AUDIENCE: env.POLICYENGINE_OTEL_GOOGLE_AUDIENCE,
+    SIMULATION_CALL_REPORT_MODAL_WORKSPACE_ID: env.MODAL_WORKSPACE_ID_VALUE,
+    SIMULATION_CALL_REPORT_MODAL_ENVIRONMENT: env.MODAL_ENVIRONMENT,
     STAGE12_ENABLED: (
       if env.DEPLOY_STAGE12_V2 == "true" then env.STAGE12_ENABLED_VALUE else "0" end
     )

.github/scripts/deploy-cloud-run-simulation-safety-check.sh

@@ -0,0 +1,132 @@
+#!/usr/bin/env bash
+# Point the safety-check Cloud Run job at the entrypoint revision that serves
+# traffic, and keep its Cloud Scheduler entry.
+#
+# The job runs `python -m policyengine_simulation_entry.safety_check`: one
+# recording pass per execution. It is updated only from a revision that was
+# promoted (or restored by a rollback), so a rejected candidate never records
+# anything. It copies the revision's image digest, service account, Modal
+# token and database URL (at the secret versions the revision pins) and its
+# plain telemetry settings; nothing else. A revision deployed before the
+# safety check existed has no job code, so its schedule is paused instead.
+#
+# With the argument `pause`, only pause the schedule: a workflow does this
+# when pointing the job at a revision failed part way, so the job never keeps
+# running the image traffic just moved away from.
+#
+# Environment: PROJECT_ID, REGION, SERVICE, and unless pausing,
+# SAFETY_CHECK_REVISION, SAFETY_CHECK_SCHEDULER_SERVICE_ACCOUNT, RUNNER_TEMP.
+set -euo pipefail
+
+fail() {
+  printf '%s\n' "$*" >&2
+  exit 1
+}
+
+pause_schedule() {
+  if gcloud scheduler jobs describe "${job}" "${scheduler_location[@]}" >/dev/null 2>&1; then
+    gcloud scheduler jobs pause "${job}" "${scheduler_location[@]}" --quiet
+  fi
+}
+
+# Every five minutes: SAFETY_CHECK_INTERVAL in
+# policyengine_simulation_entry.run_recording.
+SCHEDULE="*/5 * * * *"
+SECRETS=(SIMULATION_DATABASE_URL MODAL_TOKEN_ID MODAL_TOKEN_SECRET)
+PLAIN_VARIABLES_FILTER='^(APP_ENVIRONMENT|CLOUD_RUN_REGION|OBSERVABILITY_[A-Z_]+|OTEL_[A-Z_]+|POLICYENGINE_OTEL_GOOGLE_AUDIENCE)$'
+
+for variable_name in PROJECT_ID REGION SERVICE; do
+  [[ -n "${!variable_name:-}" ]] || fail "${variable_name} is required"
+done
+[[ "${PROJECT_ID}" =~ ^[a-z][a-z0-9-]{4,28}[a-z0-9]$ ]] || fail "PROJECT_ID is invalid"
+[[ "${REGION}" =~ ^[a-z]+-[a-z]+[0-9]+$ ]] || fail "REGION is invalid"
+[[ "${SERVICE}" =~ ^[a-z]([a-z0-9-]{0,47}[a-z0-9])?$ ]] || fail "SERVICE is invalid"
+
+job="${SERVICE}-safety-check"
+location=(--project "${PROJECT_ID}" --region "${REGION}")
+scheduler_location=(--project "${PROJECT_ID}" --location "${REGION}")
+
+if [[ "${1:-}" == "pause" ]]; then
+  pause_schedule
+  printf 'The safety check schedule for %s is paused.\n' "${SERVICE}"
+  exit 0
+fi
+
+for variable_name in SAFETY_CHECK_REVISION SAFETY_CHECK_SCHEDULER_SERVICE_ACCOUNT RUNNER_TEMP; do
+  [[ -n "${!variable_name:-}" ]] || fail "${variable_name} is required"
+done
+[[ "${SAFETY_CHECK_REVISION}" =~ ^[a-z]([a-z0-9-]{0,61}[a-z0-9])?$ ]] \
+  || fail "SAFETY_CHECK_REVISION is invalid"
+[[ "${SAFETY_CHECK_SCHEDULER_SERVICE_ACCOUNT}" =~ ^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.gserviceaccount\.com$ ]] \
+  || fail "SAFETY_CHECK_SCHEDULER_SERVICE_ACCOUNT is invalid"
+
+revision_json="$(gcloud run revisions describe "${SAFETY_CHECK_REVISION}" "${location[@]}" --format=json)"
+revision_service="$(jq -r '.metadata.labels["serving.knative.dev/service"] // ""' <<<"${revision_json}")"
+[[ "${revision_service}" == "${SERVICE}" ]] \
+  || fail "${SAFETY_CHECK_REVISION} belongs to ${revision_service:-no service}, not ${SERVICE}"
+
+# Every revision deployed with the safety check has the simulation database
+# and the expected Modal caller (SIMULATION_CALL_REPORT_MODAL_WORKSPACE_ID,
+# which the deploy has required since then); an older one has no job code.
+if ! jq -e '
+  .spec.containers[0].env // []
+  | any(.name == "SIMULATION_DATABASE_URL")
+    and any(.name == "SIMULATION_CALL_REPORT_MODAL_WORKSPACE_ID")
+' >/dev/null <<<"${revision_json}"; then
+  pause_schedule
+  printf '%s predates the safety check, so its schedule is paused.\n' "${SAFETY_CHECK_REVISION}"
+  exit 0
+fi
+
+image="$(jq -er '.status.imageDigest // .spec.containers[0].image' <<<"${revision_json}")"
+service_account="$(jq -er '.spec.serviceAccountName' <<<"${revision_json}")"
+
+job_secrets=()
+for secret_variable in "${SECRETS[@]}"; do
+  reference="$(jq -er --arg name "${secret_variable}" '
+    [.spec.containers[0].env[] | select(.name == $name) | .valueFrom.secretKeyRef
+      | select(.name != null and .key != null) | "\(.name):\(.key)"]
+    | if length == 1 then .[0] else error("not one secret reference") end
+  ' <<<"${revision_json}")" || fail "${SAFETY_CHECK_REVISION} has no pinned secret for ${secret_variable}"
+  job_secrets+=("${secret_variable}=${reference}")
+done
+job_secrets_csv="$(IFS=,; printf '%s' "${job_secrets[*]}")"
+
+environment_file="${RUNNER_TEMP}/simulation-safety-check-environment.yaml"
+trap 'rm -f "${environment_file}"' EXIT
+jq --arg filter "${PLAIN_VARIABLES_FILTER}" '
+  [.spec.containers[0].env[] | select(.value != null and (.name | test($filter)))
+    | {(.name): .value}] | add // {}
+' <<<"${revision_json}" > "${environment_file}"
+
+gcloud run jobs deploy "${job}" "${location[@]}" \
+  --image "${image}" \
+  --service-account "${service_account}" \
+  --set-secrets "${job_secrets_csv}" \
+  --env-vars-file "${environment_file}" \
+  --command /opt/venv/bin/python \
+  --args=-m,policyengine_simulation_entry.safety_check \
+  --tasks 1 \
+  --max-retries 0 \
+  --task-timeout 600s \
+  --cpu 1 \
+  --memory 512Mi \
+  --labels "serving-revision=${SAFETY_CHECK_REVISION}" \
+  --quiet
+
+schedule_action=create
+if gcloud scheduler jobs describe "${job}" "${scheduler_location[@]}" >/dev/null 2>&1; then
+  schedule_action=update
+fi
+gcloud scheduler jobs "${schedule_action}" http "${job}" "${scheduler_location[@]}" \
+  --schedule "${SCHEDULE}" \
+  --time-zone Etc/UTC \
+  --uri "https://${REGION}-run.googleapis.com/apis/run.googleapis.com/v1/namespaces/${PROJECT_ID}/jobs/${job}:run" \
+  --http-method POST \
+  --oauth-service-account-email "${SAFETY_CHECK_SCHEDULER_SERVICE_ACCOUNT}" \
+  --attempt-deadline 60s \
+  --quiet
+if [[ "${schedule_action}" == "update" ]]; then
+  gcloud scheduler jobs resume "${job}" "${scheduler_location[@]}" --quiet
+fi
+printf 'The safety check runs %s from %s.\n' "${image}" "${SAFETY_CHECK_REVISION}"

.github/scripts/simulation-entry-main-url.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+# Write the Simulation Entrypoint's main URL (its stable Cloud Run URL, which
+# reaches the revision that serves traffic) to $GITHUB_OUTPUT as `url`, for
+# the v2 Modal app's report_call_ended function.
+#
+# Environment: PROJECT_ID, REGION, SERVICE, GITHUB_OUTPUT.
+set -euo pipefail
+
+fail() {
+  printf '%s\n' "$*" >&2
+  exit 1
+}
+
+for variable_name in PROJECT_ID REGION SERVICE GITHUB_OUTPUT; do
+  [[ -n "${!variable_name:-}" ]] || fail "${variable_name} is required"
+done
+
+url="$(gcloud run services describe "${SERVICE}" \
+  --project "${PROJECT_ID}" \
+  --region "${REGION}" \
+  --format='value(status.url)')"
+[[ "${url}" =~ ^https://[a-z0-9.-]+$ ]] \
+  || fail "${SERVICE} has no https main URL: ${url:-none}"
+printf 'url=%s\n' "${url}" >> "${GITHUB_OUTPUT}"

.github/workflows/simulation-deploy.reusable.yml

@@ -279,6 +279,7 @@ jobs:
           STAGE12_DATABASE_URL_SECRET_NAME: ${{ vars.STAGE12_DATABASE_URL_SECRET_NAME }}
           MODAL_TOKEN_ID_SECRET_NAME: ${{ vars.MODAL_TOKEN_ID_SECRET_NAME }}
           MODAL_TOKEN_SECRET_SECRET_NAME: ${{ vars.MODAL_TOKEN_SECRET_SECRET_NAME }}
+          MODAL_WORKSPACE_ID_VALUE: ${{ vars.MODAL_WORKSPACE_ID }}
         run: .github/scripts/deploy-cloud-run-simulation-entry.sh
 
       - name: Resolve exact candidate metadata
@@ -670,12 +671,22 @@ jobs:
           STAGE12_CACHE_BUCKET: ${{ vars.STAGE12_CACHE_BUCKET }}
         run: ../../.github/scripts/stage12-cache-precompute.sh "${{ inputs.modal_environment }}"
 
+      # The v2 functions report to sim's main URL that a call ended.
+      - name: Find the Simulation Entrypoint's main URL
+        id: entry_url
+        env:
+          PROJECT_ID: ${{ vars.SIMULATION_ENTRYPOINT_GCP_PROJECT_ID }}
+          REGION: ${{ vars.SIMULATION_ENTRYPOINT_GCP_REGION || 'us-central1' }}
+          SERVICE: ${{ inputs.entrypoint_service }}
+        run: .github/scripts/simulation-entry-main-url.sh
+
       - name: Deploy the additional versioned v2 Modal application
         working-directory: projects/policyengine-simulation-executor
         env:
           MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
           MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
           MODAL_APP_NAME: ${{ steps.application.outputs.name }}
+          SIMULATION_ENTRY_URL: ${{ steps.entry_url.outputs.url }}
           STAGE12_EXPECT_POLICYENGINE_VERSION: ${{ needs.prepare.outputs.policyengine_version }}
           STAGE12_EXPECT_US_VERSION: ${{ needs.prepare.outputs.us_version }}
           STAGE12_EXPECT_UK_VERSION: ${{ needs.prepare.outputs.uk_version }}
@@ -1011,6 +1022,15 @@ jobs:
           SIMULATION_ENTRYPOINT_TEST_ENFORCE_PERMISSIONS: ${{ vars.SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS }}
         run: uv run --frozen pytest authenticated_tests/ -v
 
+      # Only a verified, promoted revision records outcomes in the safety
+      # check, so a rejected candidate never records anything.
+      - name: Point the safety check at the promoted revision
+        id: safety_check
+        env:
+          SAFETY_CHECK_REVISION: ${{ env.TARGET_REVISION }}
+          SAFETY_CHECK_SCHEDULER_SERVICE_ACCOUNT: ${{ vars.SIMULATION_SAFETY_CHECK_SCHEDULER_SERVICE_ACCOUNT }}
+        run: .github/scripts/deploy-cloud-run-simulation-safety-check.sh
+
       - name: Restore previous revision after failed verification
         if: ${{ failure() && steps.promote.outcome == 'success' }}
         env:
@@ -1021,6 +1041,18 @@ jobs:
           SIMULATION_ENTRYPOINT_EXPECTED_CURRENT_REVISION: ${{ env.TARGET_REVISION }}
         run: .github/scripts/set-cloud-run-simulation-entry-revision.sh
 
+      - name: Point the safety check back at the previous revision
+        id: safety_check_restore
+        if: ${{ failure() && steps.safety_check.outcome == 'failure' && env.PREVIOUS_REVISION != '' }}
+        env:
+          SAFETY_CHECK_REVISION: ${{ env.PREVIOUS_REVISION }}
+          SAFETY_CHECK_SCHEDULER_SERVICE_ACCOUNT: ${{ vars.SIMULATION_SAFETY_CHECK_SCHEDULER_SERVICE_ACCOUNT }}
+        run: .github/scripts/deploy-cloud-run-simulation-safety-check.sh
+
+      - name: Pause the safety check after a failed restore
+        if: ${{ failure() && steps.safety_check_restore.outcome == 'failure' }}
+        run: .github/scripts/deploy-cloud-run-simulation-safety-check.sh pause
+
   finalize_routing:
     name: Finalize the Modal routing publication
     # Finish the durable journal after every gate, including failed uploads

.github/workflows/simulation-rollback.yml

@@ -218,6 +218,19 @@ jobs:
             .github/scripts/set-cloud-run-simulation-entry-revision.sh
           fi
 
+      - name: Point the safety check at the restored revision
+        id: safety_check
+        if: ${{ inputs.entrypoint_revision != '' }}
+        env:
+          SAFETY_CHECK_REVISION: ${{ inputs.entrypoint_revision }}
+          SAFETY_CHECK_SCHEDULER_SERVICE_ACCOUNT: ${{ vars.SIMULATION_SAFETY_CHECK_SCHEDULER_SERVICE_ACCOUNT }}
+        run: .github/scripts/deploy-cloud-run-simulation-safety-check.sh
+
+      # The job must not keep running the image traffic just moved away from.
+      - name: Pause the safety check after a failed update
+        if: ${{ failure() && steps.safety_check.outcome == 'failure' }}
+        run: .github/scripts/deploy-cloud-run-simulation-safety-check.sh pause
+
       - name: Verify the stable service serves the restored release
         if: ${{ !inputs.resolve_pending_publication }}
         env:

changelog.d/947.added.md

@@ -0,0 +1 @@
+Added simulation runs and report runs, in migration revision `0003` of the `simulation` schema. Each run has its own status and Modal call, every status change is one conditional update, and only the instance that claims a run starts its Modal call. When a v2 Modal call ends, the new `report_call_ended` Modal function reports it on `POST /internal/modal-calls/ended`, which accepts only a Modal identity token from that function; sim then reads the call's outcome from Modal and records it on the runs. A new Cloud Run job, `<service>-safety-check`, runs every five minutes on Cloud Scheduler and records calls that ended without a report. No route creates runs yet.

changelog.d/947.changed.md

@@ -0,0 +1,3 @@
+Every Simulation Entrypoint deploy now mounts sim's own Modal token as `MODAL_TOKEN_ID` and `MODAL_TOKEN_SECRET`, no longer only when `DEPLOY_STAGE12_V2` is `true`.
+
+Every deploy now also needs, in both `beta` and `prod`, the `MODAL_TOKEN_ID_SECRET_NAME` and `MODAL_TOKEN_SECRET_SECRET_NAME` environment variables naming sim's own Modal token secrets, the `MODAL_WORKSPACE_ID` environment variable, and the `SIMULATION_SAFETY_CHECK_SCHEDULER_SERVICE_ACCOUNT` environment variable with a scheduler service account allowed to run the safety-check job. Without the first three the entrypoint deploy stops; without the last, the promotion's safety-check step fails and traffic goes back to the previous revision.