From f4c3fa195621539dd0c854498a819b1d65f5a76d Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 28 Oct 2026 00:36:10 +0400 Subject: [PATCH 1/6] Add failing tests for exception records on gateway client errors The gateway answers some invalid requests with a 400 or 404 and also calls record_exception for them, which writes an ERROR record although the request's request.completed record already reports the response as outcome client_error. These tests assert that no exception is recorded for: - an unknown country (the deploy authentication test's __authentication_probe__) on POST /simulate/economy/comparison and on POST /simulate/economy/budget-window; - a ValueError from building the bundle description on either submission endpoint; - an unknown job ID on GET /jobs/{job_id}; - an unknown batch ID on GET /budget-window-jobs/{batch_job_id}; - an unknown kind on GET /versions/{kind}. Each still expects the same status code and response body. A new test checks that a server failure (Modal's FunctionCall lookup raising during GET /jobs/{job_id}, answered with 500) still records its exception as unhandled with status 500; it passes before and after the fix. The three existing tests for the 404 responses asserted the recorded LookupError, and now assert that nothing is recorded. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_budget_window_endpoints.py | 10 +- .../tests/test_endpoints.py | 8 +- .../tests/test_exception_records.py | 139 ++++++++++++++++++ .../tests/test_simulation_job_status.py | 11 +- 4 files changed, 147 insertions(+), 21 deletions(-) create mode 100644 projects/policyengine-simulation-gateway/tests/test_exception_records.py diff --git a/projects/policyengine-simulation-gateway/tests/test_budget_window_endpoints.py b/projects/policyengine-simulation-gateway/tests/test_budget_window_endpoints.py index 875eb478c..36ccf4261 100644 --- a/projects/policyengine-simulation-gateway/tests/test_budget_window_endpoints.py +++ b/projects/policyengine-simulation-gateway/tests/test_budget_window_endpoints.py @@ -135,7 +135,7 @@ def test__given_budget_window_submission__then_returns_parent_batch_job_id( } @pytest.mark.usefixtures("mock_modal") - def test__given_unknown_budget_window_job__then_records_handled_404( + def test__given_unknown_budget_window_job__then_404_without_exception( self, client: TestClient, monkeypatch ): recorded_errors = [] @@ -151,13 +151,7 @@ def test__given_unknown_budget_window_job__then_records_handled_404( assert response.json()["detail"] == ( "Budget-window job not found: missing-batch" ) - assert str(recorded_errors[0][0]) == ( - "Budget-window job not found: missing-batch" - ) - assert recorded_errors[0][1] == { - "handled": True, - "status_code": 404, - } + assert recorded_errors == [] def test__given_parent_lookup_failure__then_degraded_poll_is_not_an_error( self, mock_modal, client: TestClient, monkeypatch diff --git a/projects/policyengine-simulation-gateway/tests/test_endpoints.py b/projects/policyengine-simulation-gateway/tests/test_endpoints.py index 1e4bf293b..1c4b9f872 100644 --- a/projects/policyengine-simulation-gateway/tests/test_endpoints.py +++ b/projects/policyengine-simulation-gateway/tests/test_endpoints.py @@ -226,7 +226,7 @@ def test__given_active_state__then_policyengine_versions_endpoint_works( assert response.json()["latest"] == "4.10.0" @pytest.mark.usefixtures("mock_modal") - def test__given_unknown_version_kind__then_records_handled_404( + def test__given_unknown_version_kind__then_404_without_exception( self, client: TestClient, monkeypatch ): recorded_errors = [] @@ -240,11 +240,7 @@ def test__given_unknown_version_kind__then_records_handled_404( assert response.status_code == 404 assert response.json()["detail"] == "Unknown version kind: fr" - assert str(recorded_errors[0][0]) == "Unknown version kind: fr" - assert recorded_errors[0][1] == { - "handled": True, - "status_code": 404, - } + assert recorded_errors == [] def test__given_no_active_state__then_versions_fall_back_to_old_dicts( self, mock_modal, client: TestClient diff --git a/projects/policyengine-simulation-gateway/tests/test_exception_records.py b/projects/policyengine-simulation-gateway/tests/test_exception_records.py new file mode 100644 index 000000000..e6485b408 --- /dev/null +++ b/projects/policyengine-simulation-gateway/tests/test_exception_records.py @@ -0,0 +1,139 @@ +"""Client errors the gateway answers with 4xx record no exception; 5xx still do. + +``ObservabilityRuntime.record_exception`` always writes an ERROR record. A +request the gateway answers with a 4xx already gets a ``request.completed`` +record whose outcome is ``client_error``, so an exception record for it adds +only a false alarm to the daily error report. A server failure (5xx) keeps +its exception record. +""" + +from typing import NamedTuple + +import pytest +from fastapi.testclient import TestClient +from policyengine_simulation_gateway import job_creation_routes + +# The country the deploy pipeline's authentication test sends, expecting 400. +AUTHENTICATION_PROBE_COUNTRY = "__authentication_probe__" + + +class Submission(NamedTuple): + """A submission endpoint and the request fields it needs besides country.""" + + path: str + request_fields: dict[str, object] + + +SUBMISSIONS = [ + pytest.param( + Submission( + path="/simulate/economy/comparison", + request_fields={"scope": "macro", "reform": {}}, + ), + id="comparison", + ), + pytest.param( + Submission( + path="/simulate/economy/budget-window", + request_fields={ + "region": "us", + "scope": "macro", + "reform": {}, + "start_year": "2026", + "window_size": 2, + }, + ), + id="budget_window", + ), +] + +RecordedException = tuple[BaseException, dict[str, object]] + + +@pytest.fixture +def recorded_exceptions( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> list[RecordedException]: + """Replace the runtime's ``record_exception`` with one that keeps each call.""" + recorded: list[RecordedException] = [] + + def record_exception(error: BaseException, **keyword_arguments: object) -> None: + recorded.append((error, keyword_arguments)) + + monkeypatch.setattr( + client.app.state.policyengine_observability, + "record_exception", + record_exception, + ) + return recorded + + +@pytest.mark.parametrize("submission", SUBMISSIONS) +def test__given_unknown_country__then_returns_400_without_exception_record( + mock_modal, + client: TestClient, + recorded_exceptions: list[RecordedException], + submission: Submission, +): + response = client.post( + submission.path, + json={"country": AUTHENTICATION_PROBE_COUNTRY, **submission.request_fields}, + ) + + assert response.status_code == 400 + assert response.json() == { + "detail": f"Unknown country: {AUTHENTICATION_PROBE_COUNTRY}" + } + assert recorded_exceptions == [] + assert mock_modal["func"].calls == [] + + +@pytest.mark.parametrize("submission", SUBMISSIONS) +def test__given_bundle_lookup_error__then_returns_400_without_exception_record( + mock_modal, + client: TestClient, + recorded_exceptions: list[RecordedException], + monkeypatch: pytest.MonkeyPatch, + submission: Submission, +): + bundle_lookup_message = "The requested bundle has no dataset for this request" + + def reject_bundle_lookup(*_arguments: object) -> None: + raise ValueError(bundle_lookup_message) + + monkeypatch.setattr( + job_creation_routes, "build_policyengine_bundle", reject_bundle_lookup + ) + + response = client.post( + submission.path, json={"country": "us", **submission.request_fields} + ) + + assert response.status_code == 400 + assert response.json() == {"detail": bundle_lookup_message} + assert recorded_exceptions == [] + assert mock_modal["func"].calls == [] + + +def test__given_modal_lookup_failure__then_job_status_records_server_exception( + mock_modal, + client: TestClient, + recorded_exceptions: list[RecordedException], +): + submit_response = client.post( + "/simulate/economy/comparison", + json={"country": "us", "scope": "macro", "reform": {}}, + ) + job_id = submit_response.json()["job_id"] + lookup_failure = RuntimeError("Modal control plane unavailable") + mock_modal["function_call"].from_id_errors[job_id] = lookup_failure + client_receiving_server_errors = TestClient( + client.app, raise_server_exceptions=False + ) + + response = client_receiving_server_errors.get(f"/jobs/{job_id}") + + assert response.status_code == 500 + assert recorded_exceptions == [ + (lookup_failure, {"handled": False, "status_code": 500}), + ] diff --git a/projects/policyengine-simulation-gateway/tests/test_simulation_job_status.py b/projects/policyengine-simulation-gateway/tests/test_simulation_job_status.py index 0f972c26f..627748b50 100644 --- a/projects/policyengine-simulation-gateway/tests/test_simulation_job_status.py +++ b/projects/policyengine-simulation-gateway/tests/test_simulation_job_status.py @@ -79,14 +79,15 @@ def test__given_submitted_job_with_header__then_polling_echoes_header( assert response.headers[OBSERVABILITY_ID_HEADER] == OBSERVABILITY_ID @pytest.mark.usefixtures("mock_modal") - def test__given_unknown_job_id__then_polling_returns_404( + def test__given_unknown_job_id__then_polling_returns_404_without_exception( self, client: TestClient, monkeypatch ): """Return 404 for a job ID the gateway never issued, without asking Modal. Given a job id that the gateway never issued When polling job status - Then the gateway returns 404 before asking Modal for a call result. + Then the gateway returns 404 before asking Modal for a call result, + and records no exception, because the 404 is a client error. """ recorded_errors = [] monkeypatch.setattr( @@ -103,11 +104,7 @@ def test__given_unknown_job_id__then_polling_returns_404( assert response.status_code == 404 assert response.json()["detail"] == "Job not found: unknown-job-id" assert OBSERVABILITY_ID_HEADER not in response.headers - assert str(recorded_errors[0][0]) == "Job not found: unknown-job-id" - assert recorded_errors[0][1] == { - "handled": True, - "status_code": 404, - } + assert recorded_errors == [] def test__given_lazy_modal_call_without_metadata__then_polling_returns_404( self, mock_modal, client: TestClient From 3a893bbdeb0f5853778da3cb5675d1aa141bb4c3 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 28 Oct 2026 00:39:30 +0400 Subject: [PATCH 2/6] Stop recording exceptions for gateway client errors ObservabilityRuntime.record_exception always writes an ERROR record, and the daily error report counts every ERROR record. The gateway called it for requests it then answered with a client error, although each request's request.completed record already reports the response with outcome client_error. Most of these records came from the deploy pipeline's authentication test, which sends the country __authentication_probe__ and expects 400, and from a staging check that polls an unknown job ID. The gateway no longer records an exception before it returns: - 400 from POST /simulate/economy/comparison and POST /simulate/economy/budget-window when route resolution, or the bundle and SPM lookup, raises a ValueError (four calls); - 404 from GET /jobs/{job_id} when the gateway has no metadata for the job ID; - 404 from GET /budget-window-jobs/{batch_job_id} when no batch state or seed exists for the batch ID; - 404 from GET /versions/{kind} for an unknown kind. The last three called routes_common.record_not_found, which has no callers left and is removed. Status codes and response bodies are unchanged, and no other log replaces the records. Still recorded: the 500 when Modal's FunctionCall lookup fails during GET /jobs/{job_id}, failed-job responses through log_and_redact_exception, and the 404 for a job whose gateway metadata exists but whose Modal call is missing or whose output has expired, which is not clearly a client error. Co-Authored-By: Claude Opus 5.5 --- .../policyengine_simulation_gateway/job_creation_routes.py | 4 ---- .../src/policyengine_simulation_gateway/job_status_routes.py | 3 --- .../src/policyengine_simulation_gateway/routes_common.py | 5 ----- .../src/policyengine_simulation_gateway/version_routes.py | 2 -- 4 files changed, 14 deletions(-) diff --git a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/job_creation_routes.py b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/job_creation_routes.py index 36bbc4ed3..e2c1fcf5b 100644 --- a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/job_creation_routes.py +++ b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/job_creation_routes.py @@ -226,7 +226,6 @@ async def submit_simulation( ) _require_route_serves_breakdown(request, route) except ValueError as e: - runtime.record_exception(e, handled=True, status_code=400) raise HTTPException(status_code=400, detail=str(e)) from e runtime.set_context( @@ -259,7 +258,6 @@ async def submit_simulation( errors=[detail.model_dump()], headers=observability_headers(observability_id), ) - runtime.record_exception(exc, handled=True, status_code=400) raise HTTPException(status_code=400, detail=str(exc)) from exc if request.spm is not None: @@ -336,7 +334,6 @@ async def submit_budget_window_batch( request.policyengine_version, ) except ValueError as e: - runtime.record_exception(e, handled=True, status_code=400) raise HTTPException(status_code=400, detail=str(e)) from e runtime.set_context( @@ -360,7 +357,6 @@ async def submit_budget_window_batch( errors=[detail.model_dump()], headers=observability_headers(observability_id), ) - runtime.record_exception(exc, handled=True, status_code=400) raise HTTPException(status_code=400, detail=str(exc)) from exc with runtime.span(BUDGET_WINDOW_STAGES.name(Stage.REQUEST_PARSE)): payload = _build_budget_window_parent_payload( diff --git a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/job_status_routes.py b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/job_status_routes.py index 986440248..daf0cec25 100644 --- a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/job_status_routes.py +++ b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/job_status_routes.py @@ -46,7 +46,6 @@ logger, observability_headers, observability_runtime, - record_not_found, router, ) @@ -122,7 +121,6 @@ async def get_job_status( with runtime.span(ANNUAL_IMPACT_STAGES.name(Stage.MODAL_JOB_METADATA_READ)): job_metadata = job_metadata_store().get(job_id) if job_metadata is None: - record_not_found(runtime, f"Job not found: {job_id}") raise HTTPException(status_code=404, detail=f"Job not found: {job_id}") observability_id = bind_observability_id( request, @@ -226,7 +224,6 @@ async def get_budget_window_job_status( with runtime.span(BUDGET_WINDOW_STAGES.name(Stage.BUDGET_WINDOW_STATE_LOAD)): seed_state = get_batch_job_seed(batch_job_id) if seed_state is None: - record_not_found(runtime, f"Budget-window job not found: {batch_job_id}") raise HTTPException( status_code=404, detail=f"Budget-window job not found: {batch_job_id}" ) diff --git a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/routes_common.py b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/routes_common.py index 7bd809b03..140ea2d0d 100644 --- a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/routes_common.py +++ b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/routes_common.py @@ -25,11 +25,6 @@ def observability_runtime(request: Request) -> ObservabilityRuntime: return runtime -def record_not_found(runtime: ObservabilityRuntime, message: str) -> None: - """Record a handled not-found lookup (status 404) on the request's telemetry.""" - runtime.record_exception(LookupError(message), handled=True, status_code=404) - - def bind_observability_id( http_request: Request, runtime: ObservabilityRuntime, diff --git a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/version_routes.py b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/version_routes.py index 5dc1e8c5d..44f4e3f44 100644 --- a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/version_routes.py +++ b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/version_routes.py @@ -29,7 +29,6 @@ from policyengine_simulation_gateway.routes_common import ( logger, observability_runtime, - record_not_found, router, ) @@ -92,7 +91,6 @@ async def get_country_versions(kind: str, request: Request) -> VersionMap: runtime = observability_runtime(request) kind_lower = kind.lower() if kind_lower not in SUPPORTED_ROUTE_KINDS: - record_not_found(runtime, f"Unknown version kind: {kind}") raise HTTPException(status_code=404, detail=f"Unknown version kind: {kind}") with runtime.span(ANNUAL_IMPACT_STAGES.name(Stage.ROUTE_RESOLUTION)): From 4b86bf50bac06466dbc4946ed06c16792d50c1ef Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 28 Oct 2026 01:01:07 +0400 Subject: [PATCH 3/6] Add failing tests for a redacted 500 on bundle lookup errors Both submission handlers answer a ValueError or HuggingFaceDatasetReferenceError without an SPM error code, raised while describing the route's bundle or resolving SPM settings, with a 400 that repeats the exception text. No request input reaches that branch: coded SPM errors take the structured 400 response, and the bundle step reads the deployed bundle manifest, selected by the already-checked country and version. Only bad manifest data, such as a malformed dataset reference, can raise there, so it is a server failure. The bundle lookup tests for POST /simulate/economy/comparison and POST /simulate/economy/budget-window now expect: - status 500 with status "failed", a null result, no structured errors, and the error "Simulation failed (correlation_id=<32 hex digits>)"; - no part of the exception text in the response; - the observability ID header; - the exception recorded once as handled with status 500; - no Modal spawn. Both fail before the fix: the handlers return 400. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_exception_records.py | 31 +++++++++++++++---- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/projects/policyengine-simulation-gateway/tests/test_exception_records.py b/projects/policyengine-simulation-gateway/tests/test_exception_records.py index e6485b408..52ed4248b 100644 --- a/projects/policyengine-simulation-gateway/tests/test_exception_records.py +++ b/projects/policyengine-simulation-gateway/tests/test_exception_records.py @@ -5,13 +5,19 @@ record whose outcome is ``client_error``, so an exception record for it adds only a false alarm to the daily error report. A server failure (5xx) keeps its exception record. + +An uncoded error while describing the route's bundle comes from the deployed +bundle manifest, not from the request, so it is a server failure. """ +import re from typing import NamedTuple import pytest from fastapi.testclient import TestClient +from policyengine_simulation_contract.hf_dataset import HuggingFaceDatasetReferenceError from policyengine_simulation_gateway import job_creation_routes +from policyengine_simulation_observability.identifiers import OBSERVABILITY_ID_HEADER # The country the deploy pipeline's authentication test sends, expecting 400. AUTHENTICATION_PROBE_COUNTRY = "__authentication_probe__" @@ -89,17 +95,20 @@ def test__given_unknown_country__then_returns_400_without_exception_record( @pytest.mark.parametrize("submission", SUBMISSIONS) -def test__given_bundle_lookup_error__then_returns_400_without_exception_record( +def test__given_bundle_lookup_error__then_records_it_and_returns_redacted_500( mock_modal, client: TestClient, recorded_exceptions: list[RecordedException], monkeypatch: pytest.MonkeyPatch, submission: Submission, ): - bundle_lookup_message = "The requested bundle has no dataset for this request" + malformed_dataset_reference = "hf://policyengine/internal-repo/missing.h5" + manifest_error = HuggingFaceDatasetReferenceError( + f"Malformed dataset reference in bundle manifest: {malformed_dataset_reference}" + ) def reject_bundle_lookup(*_arguments: object) -> None: - raise ValueError(bundle_lookup_message) + raise manifest_error monkeypatch.setattr( job_creation_routes, "build_policyengine_bundle", reject_bundle_lookup @@ -109,9 +118,19 @@ def reject_bundle_lookup(*_arguments: object) -> None: submission.path, json={"country": "us", **submission.request_fields} ) - assert response.status_code == 400 - assert response.json() == {"detail": bundle_lookup_message} - assert recorded_exceptions == [] + assert response.status_code == 500 + body = response.json() + assert body["status"] == "failed" + assert body["result"] is None + assert "errors" not in body + assert re.fullmatch( + r"Simulation failed \(correlation_id=[0-9a-f]{32}\)", body["error"] + ) + assert malformed_dataset_reference not in response.text + assert OBSERVABILITY_ID_HEADER in response.headers + assert recorded_exceptions == [ + (manifest_error, {"handled": True, "status_code": 500}), + ] assert mock_modal["func"].calls == [] From 0715a7a5927b1647f87e24961ca226c39daee973 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 28 Oct 2026 01:01:51 +0400 Subject: [PATCH 4/6] Return a recorded, redacted 500 for bundle lookup errors In submit_simulation and submit_budget_window_batch, an uncoded ValueError or HuggingFaceDatasetReferenceError raised while describing the route's bundle or resolving SPM settings used to become a 400 whose detail was the exception text. Only bad data in the deployed bundle manifest can raise there, so both handlers now treat it like the gateway's other server failures: - log_and_redact_exception logs the exception with its traceback, records it as handled with status 500, and returns "Simulation failed (correlation_id=)"; - failed_job_response returns that message as a 500 with status "failed", without structured errors, with the observability ID header. The caller no longer sees the exception text. The log scopes are simulation_submission and budget_window_submission, with the resolved app name as context. A coded SPM error still returns 400 with structured errors, unchanged. A ValueError from route resolution, such as an unknown country, still returns 400 without an exception record. Co-Authored-By: Claude Opus 5.5 --- .../job_creation_routes.py | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/job_creation_routes.py b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/job_creation_routes.py index e2c1fcf5b..2ca0cbc9c 100644 --- a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/job_creation_routes.py +++ b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/job_creation_routes.py @@ -35,6 +35,7 @@ resolve_spm_selection, spm_error_detail, ) +from policyengine_simulation_observability.errors import log_and_redact_exception from policyengine_simulation_observability.identifiers import ( OBSERVABILITY_ID_HEADER, resolve_observability_id, @@ -258,7 +259,20 @@ async def submit_simulation( errors=[detail.model_dump()], headers=observability_headers(observability_id), ) - raise HTTPException(status_code=400, detail=str(exc)) from exc + # No request input reaches this point: the country and version are + # already checked, and every request-driven SPM error has a code. The + # error comes from the deployed bundle manifest, so it is a server + # failure, and the caller gets a redacted message. + redacted_error = log_and_redact_exception( + exc, + runtime=runtime, + scope="simulation_submission", + context={"resolved_app_name": route.app_name}, + ) + return failed_job_response( + error=redacted_error, + headers=observability_headers(observability_id), + ) if request.spm is not None: payload["spm"] = request.spm.model_dump(mode="json") @@ -357,7 +371,20 @@ async def submit_budget_window_batch( errors=[detail.model_dump()], headers=observability_headers(observability_id), ) - raise HTTPException(status_code=400, detail=str(exc)) from exc + # No request input reaches this point: the country and version are + # already checked, and every request-driven SPM error has a code. The + # error comes from the deployed bundle manifest, so it is a server + # failure, and the caller gets a redacted message. + redacted_error = log_and_redact_exception( + exc, + runtime=runtime, + scope="budget_window_submission", + context={"resolved_app_name": route.app_name}, + ) + return failed_job_response( + error=redacted_error, + headers=observability_headers(observability_id), + ) with runtime.span(BUDGET_WINDOW_STAGES.name(Stage.REQUEST_PARSE)): payload = _build_budget_window_parent_payload( request, From 0965c74859306ec6d7691d92fb38f058a2fcabed Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 28 Oct 2026 01:06:17 +0400 Subject: [PATCH 5/6] Declare the 500 on both submission routes POST /simulate/economy/comparison and POST /simulate/economy/budget-window can now answer 500 with a redacted error, so both routes declare it as "Server failure (redacted error with a correlation ID)", next to the existing 200, 400 and 422: - in the gateway's OpenAPI generator, with tests/golden/openapi.json regenerated (the only change is the two 500 entries); - in the entrypoint's route declarations, which publish the schema the simulation client is generated from. The entrypoint's contract test compares its OpenAPI document with the gateway's golden file, so the two descriptions are identical. The 500 has no response model, like the 500 on the status routes, because an unhandled server error still returns a plain-text body. The generated client is not committed; clients are published from the deployed commit. Regenerated locally, only the two submission modules change: a 500 is now a declared status that parses to None, where it used to raise UnexpectedStatus when raise_on_unexpected_status is set. Co-Authored-By: Claude Opus 5.5 --- .../src/policyengine_simulation_entry/app.py | 6 ++++++ .../src/policyengine_simulation_gateway/generate_openapi.py | 6 ++++++ .../tests/golden/openapi.json | 6 ++++++ 3 files changed, 18 insertions(+) diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py index be3a699ce..759610fc6 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py @@ -329,6 +329,9 @@ def _register_simulation_routes( "description": "Invalid request or SPM selection", "model": SimulationErrorResponse, }, + 500: { + "description": "Server failure (redacted error with a correlation ID)" + }, }, dependencies=[Depends(submitter)], ) @@ -368,6 +371,9 @@ async def submit_comparison( "description": "Invalid request or SPM selection", "model": SimulationErrorResponse, }, + 500: { + "description": "Server failure (redacted error with a correlation ID)" + }, }, dependencies=[Depends(submitter)], ) diff --git a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/generate_openapi.py b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/generate_openapi.py index f61c2cf03..374019dd3 100644 --- a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/generate_openapi.py +++ b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/generate_openapi.py @@ -40,6 +40,9 @@ def create_openapi_app() -> FastAPI: responses={ 200: {"description": "Job submitted successfully"}, 400: {"description": "Invalid request (unknown country/version)"}, + 500: { + "description": "Server failure (redacted error with a correlation ID)" + }, }, ) async def submit_simulation(request: SimulationRequest) -> JobSubmitResponse: @@ -56,6 +59,9 @@ async def submit_simulation(request: SimulationRequest) -> JobSubmitResponse: responses={ 200: {"description": "Budget-window batch submitted successfully"}, 400: {"description": "Invalid request (unknown country/version/year)"}, + 500: { + "description": "Server failure (redacted error with a correlation ID)" + }, }, ) async def submit_budget_window_batch( diff --git a/projects/policyengine-simulation-gateway/tests/golden/openapi.json b/projects/policyengine-simulation-gateway/tests/golden/openapi.json index 9a8e102ea..ae9c9d9c5 100644 --- a/projects/policyengine-simulation-gateway/tests/golden/openapi.json +++ b/projects/policyengine-simulation-gateway/tests/golden/openapi.json @@ -35,6 +35,9 @@ "400": { "description": "Invalid request (unknown country/version)" }, + "500": { + "description": "Server failure (redacted error with a correlation ID)" + }, "422": { "description": "Validation Error", "content": { @@ -77,6 +80,9 @@ "400": { "description": "Invalid request (unknown country/version/year)" }, + "500": { + "description": "Server failure (redacted error with a correlation ID)" + }, "422": { "description": "Validation Error", "content": { From 9741269281e757d91f5bfdb887822d4402903d77 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:56:58 +0400 Subject: [PATCH 6/6] Replace the changelog entry with a changelog fragment main replaced changelog_entry.yaml with Towncrier fragments in changelog.d/ (rule sim.process.changelog-fragment), so the rebase drops this pull request's changelog_entry.yaml item and this commit adds changelog.d/962.changed.md, named after the tracking issue, #962. Its type, changed, puts it under Changed and raises the patch version. The fragment describes the same three changes as the old item: the gateway no longer records an exception for the 400 and 404 client errors it returns; an uncoded ValueError or HuggingFaceDatasetReferenceError from the bundle and SPM lookup now returns a recorded, redacted 500; and both economy submission routes declare that 500. Co-Authored-By: Claude Opus 5.5 --- changelog.d/962.changed.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changelog.d/962.changed.md diff --git a/changelog.d/962.changed.md b/changelog.d/962.changed.md new file mode 100644 index 000000000..d2762abca --- /dev/null +++ b/changelog.d/962.changed.md @@ -0,0 +1,3 @@ +The gateway no longer records an exception, which `policyengine-observability` writes at ERROR severity, for a request it answers with a 400 or 404 client error: a `ValueError` from route resolution on `POST /simulate/economy/comparison` or `POST /simulate/economy/budget-window`, such as the unknown country sent by the authentication test that each deployment runs, an unknown job ID on `GET /jobs/{job_id}`, an unknown batch ID on `GET /budget-window-jobs/{batch_job_id}`, or an unknown kind on `GET /versions/{kind}`. Status codes and response bodies are unchanged, and each request's `request.completed` record still reports the response with outcome `client_error`. Server errors, and the 404 for a job whose gateway metadata exists but whose Modal call is missing or whose output has expired, still record their exceptions. + +On both economy submission routes, a `ValueError` or `HuggingFaceDatasetReferenceError` without an SPM error code, raised while describing the route's bundle or resolving its SPM settings, now returns 500 instead of 400. Only bad data in the deployed bundle manifest can raise it. The gateway logs the exception with its traceback and records it as a handled server error, and the response has `status: "failed"` and the redacted error `Simulation failed (correlation_id=...)` instead of the exception text. A coded SPM error still returns 400 with structured `errors`. Both routes declare the 500 in the gateway's OpenAPI document and in the entrypoint's, so the generated client's two submission functions handle it as a declared status.