diff --git a/changelog.d/record-flask-exceptions.changed.md b/changelog.d/record-flask-exceptions.changed.md new file mode 100644 index 000000000..0a8e6bef8 --- /dev/null +++ b/changelog.d/record-flask-exceptions.changed.md @@ -0,0 +1 @@ +Unhandled exceptions in Flask routes are now recorded in telemetry with their exception type and stack, on an exception record and on the request's completion record, so error reports can group them. This requires policyengine-observability 3.0.3. diff --git a/gcp/observability/README.md b/gcp/observability/README.md index a7d384dd3..695a57ff7 100644 --- a/gcp/observability/README.md +++ b/gcp/observability/README.md @@ -113,7 +113,7 @@ The infrastructure was applied and verified on 2026-09-22: - The alert policies have no notification channels, so they record incidents without sending email, Slack, or paging notifications. -The consumer services require `policyengine-observability` 3.0.2. Record the +The consumer services require `policyengine-observability` 3.0.3. Record the deployed consumer revisions and a representative cost and volume observation interval after the API v1 rollout. diff --git a/policyengine_api/observability/__init__.py b/policyengine_api/observability/__init__.py index c7c1121f3..e4442cf82 100644 --- a/policyengine_api/observability/__init__.py +++ b/policyengine_api/observability/__init__.py @@ -1,5 +1,17 @@ """API observability runtime, identifiers, and registered stage plans.""" -from .runtime import _build_runtime, get_runtime, runtime, set_runtime_context +from .runtime import ( + _build_runtime, + get_runtime, + record_unhandled_exception, + runtime, + set_runtime_context, +) -__all__ = ["_build_runtime", "get_runtime", "runtime", "set_runtime_context"] +__all__ = [ + "_build_runtime", + "get_runtime", + "record_unhandled_exception", + "runtime", + "set_runtime_context", +] diff --git a/policyengine_api/observability/runtime.py b/policyengine_api/observability/runtime.py index b01840b18..edaad52cc 100644 --- a/policyengine_api/observability/runtime.py +++ b/policyengine_api/observability/runtime.py @@ -78,3 +78,10 @@ def set_runtime_context(**attributes: object) -> None: # Telemetry must never affect application behavior. with contextlib.suppress(Exception): runtime.set_context(**attributes) + + +def record_unhandled_exception(error: Exception, *, status_code: int = 500) -> None: + """Record an exception that reached a catch-all error handler.""" + # Telemetry must never affect application behavior. + with contextlib.suppress(Exception): + runtime.record_exception(error, handled=False, status_code=status_code) diff --git a/policyengine_api/routes/error_routes.py b/policyengine_api/routes/error_routes.py index ab6005351..067d25332 100644 --- a/policyengine_api/routes/error_routes.py +++ b/policyengine_api/routes/error_routes.py @@ -5,6 +5,7 @@ HTTPException, ) +from policyengine_api.observability import record_unhandled_exception from policyengine_api.response_factory import _make_error_response error_bp = Blueprint("error", __name__) @@ -49,4 +50,5 @@ def response_http_exception(error: HTTPException) -> Response: @error_bp.app_errorhandler(Exception) def response_generic_error(error: Exception) -> Response: """Return an unhandled exception as a 500 JSON error body carrying its message.""" + record_unhandled_exception(error, status_code=500) return _make_error_response(error, 500, result=None) diff --git a/pyproject.toml b/pyproject.toml index a6c52ccf3..5b5a04ef1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ dependencies = [ "microdf_python>=1.0.0", "openai", "packaging>=24,<27", - "policyengine-observability[flask,google,httpx,otlp-grpc]>=3.0.2,<4", + "policyengine-observability[flask,google,httpx,otlp-grpc]>=3.0.3,<4", "policyengine_canada==0.96.3", "policyengine-ng==0.5.1", "policyengine-il==0.1.0", diff --git a/tests/unit/routes/test_error_routes.py b/tests/unit/routes/test_error_routes.py new file mode 100644 index 000000000..e83ab4a97 --- /dev/null +++ b/tests/unit/routes/test_error_routes.py @@ -0,0 +1,169 @@ +"""The Flask catch-all error handler records unhandled exceptions in telemetry.""" + +import importlib +from dataclasses import replace + +import pytest +from flask import Flask +from policyengine_observability import ( + CustomLogDestination, + LoggingConfig, + OTelConfig, + configure, + instrument_flask, +) + +from policyengine_api.observability import runtime +from policyengine_api.routes import error_routes + +# policyengine_api.observability re-exports the runtime object under the name of +# its submodule, so the submodule whose global record_unhandled_exception reads +# is imported by its full name. +runtime_module = importlib.import_module("policyengine_api.observability.runtime") + +ENGINE_FAILURE = "engine exploded" +EXISTING_500_BODY = {"status": "error", "message": ENGINE_FAILURE, "result": None} + + +class _CollectedRecords: + """Keeps every structured record a runtime delivers, in delivery order.""" + + def __init__(self) -> None: + self.records: list[dict[str, object]] = [] + + def write(self, record: dict[str, object]) -> None: + self.records.append(record) + + def with_event_name(self, event_name: str) -> list[dict[str, object]]: + return [ + record for record in self.records if record.get("event.name") == event_name + ] + + +@pytest.fixture +def collected_records(): + return _CollectedRecords() + + +@pytest.fixture +def collecting_runtime(monkeypatch, collected_records): + """Point record_unhandled_exception at a real runtime that keeps its records.""" + observability_runtime = configure( + replace( + runtime.config, + logging=LoggingConfig( + destinations=( + CustomLogDestination( + name="collected-records", + writer_factory=lambda: collected_records, + delivery="inline", + ), + ), + ), + otel=OTelConfig(enabled=False), + ) + ) + monkeypatch.setattr(runtime_module, "runtime", observability_runtime) + yield observability_runtime + observability_runtime.shutdown() + + +def _make_client(observability_runtime=None): + app = Flask(__name__) + app.config["TESTING"] = True + if observability_runtime is not None: + # Same order as the API app: instrument Flask, then register handlers. + instrument_flask(app, observability_runtime) + app.register_blueprint(error_routes.error_bp) + + @app.route("/explode") + def explode(): + raise RuntimeError(ENGINE_FAILURE) + + return app.test_client() + + +def test_unhandled_exception_is_recorded_and_still_returns_500(monkeypatch): + recorded_calls = [] + monkeypatch.setattr( + error_routes, + "record_unhandled_exception", + lambda error, **options: recorded_calls.append((error, options)), + ) + + response = _make_client().get("/explode") + + assert response.status_code == 500 + assert response.get_json() == EXISTING_500_BODY + assert len(recorded_calls) == 1 + recorded_error, recorded_options = recorded_calls[0] + assert isinstance(recorded_error, RuntimeError) + assert str(recorded_error) == ENGINE_FAILURE + assert recorded_options == {"status_code": 500} + + +def test_unhandled_exception_adds_one_exception_record_to_its_request( + collecting_runtime, collected_records +): + response = _make_client(collecting_runtime).get("/explode") + + assert response.status_code == 500 + assert response.get_json() == EXISTING_500_BODY + exception_records = collected_records.with_event_name("exception.recorded") + completed_records = collected_records.with_event_name("request.completed") + assert len(exception_records) == 1 + assert len(completed_records) == 1 + exception_record = exception_records[0] + assert exception_record["error.type"] == "RuntimeError" + assert exception_record["error.message"] == ENGINE_FAILURE + assert "raise RuntimeError(ENGINE_FAILURE)" in exception_record["error.stack"] + assert exception_record["error.handled"] is False + assert exception_record["http.response.status_code"] == 500 + assert exception_record["http.route"] == "/explode" + assert exception_record["request.id"] == completed_records[0]["request.id"] + + +def test_unhandled_exception_is_on_its_request_completion_record( + collecting_runtime, collected_records +): + _make_client(collecting_runtime).get("/explode") + + completed_record = collected_records.with_event_name("request.completed")[0] + assert completed_record["outcome"] == "error" + assert completed_record["http.response.status_code"] == 500 + assert completed_record.get("error.type") == "RuntimeError" + assert completed_record.get("error.message") == ENGINE_FAILURE + assert "raise RuntimeError(ENGINE_FAILURE)" in completed_record.get( + "error.stack", "" + ) + + +def test_not_found_is_not_recorded_as_unhandled_exception(monkeypatch): + recorded_calls = [] + monkeypatch.setattr( + error_routes, + "record_unhandled_exception", + lambda error, **options: recorded_calls.append((error, options)), + ) + + response = _make_client().get("/missing") + + assert response.status_code == 404 + assert recorded_calls == [] + + +def test_recording_failure_leaves_500_response_unchanged(monkeypatch): + attempted_errors = [] + + def fail_to_record(error, **_options): + attempted_errors.append(error) + message = "observability unavailable" + raise RuntimeError(message) + + monkeypatch.setattr(runtime, "record_exception", fail_to_record) + + response = _make_client().get("/explode") + + assert len(attempted_errors) == 1 + assert response.status_code == 500 + assert response.get_json() == EXISTING_500_BODY diff --git a/tests/unit/test_observability_runtime.py b/tests/unit/test_observability_runtime.py index 70740dfc9..74db7c72c 100644 --- a/tests/unit/test_observability_runtime.py +++ b/tests/unit/test_observability_runtime.py @@ -7,11 +7,17 @@ from policyengine_api.observability import ( _build_runtime, + record_unhandled_exception, runtime, set_runtime_context, ) +def _fail_observability(*_arguments, **_options): + message = "observability unavailable" + raise RuntimeError(message) + + def test_runtime_uses_consumer_owned_identity_and_stdout(monkeypatch): monkeypatch.setenv("OTEL_SDK_DISABLED", "true") monkeypatch.setenv("OTEL_TRACES_SAMPLER_ARG", "0.01") @@ -48,3 +54,23 @@ def test_runtime_context_failure_does_not_escape(monkeypatch): ) set_runtime_context(country_id="us") + + +def test_unhandled_exception_is_recorded_as_unhandled_server_error(monkeypatch): + recorded_calls = [] + monkeypatch.setattr( + runtime, + "record_exception", + lambda error, **options: recorded_calls.append((error, options)), + ) + error = RuntimeError("engine exploded") + + record_unhandled_exception(error) + + assert recorded_calls == [(error, {"handled": False, "status_code": 500})] + + +def test_unhandled_exception_recording_failure_does_not_escape(monkeypatch): + monkeypatch.setattr(runtime, "record_exception", _fail_observability) + + record_unhandled_exception(RuntimeError("engine exploded")) diff --git a/uv.lock b/uv.lock index 558bd32cf..0b851e2b1 100644 --- a/uv.lock +++ b/uv.lock @@ -2849,7 +2849,7 @@ requires-dist = [ { name = "policyengine-canada", specifier = "==0.96.3" }, { name = "policyengine-il", specifier = "==0.1.0" }, { name = "policyengine-ng", specifier = "==0.5.1" }, - { name = "policyengine-observability", extras = ["flask", "google", "httpx", "otlp-grpc"], specifier = ">=3.0.2,<4" }, + { name = "policyengine-observability", extras = ["flask", "google", "httpx", "otlp-grpc"], specifier = ">=3.0.3,<4" }, { name = "psycopg", extras = ["binary"], specifier = ">=3.3,<4" }, { name = "pydantic" }, { name = "pyjwt", extras = ["crypto"], specifier = ">=2.10,<3" }, @@ -2953,11 +2953,11 @@ wheels = [ [[package]] name = "policyengine-observability" -version = "3.0.2" +version = "3.0.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/74/2a/0c261c8ca693bb3a13d7fe837d3f89f7fdda245496d4b3e92eefed3c972e/policyengine_observability-3.0.2.tar.gz", hash = "sha256:554f907e43d8eeb274c1298fbe990b1626ac45f194c49995ddde091ddcdc2a64", size = 125718, upload-time = "2026-09-30T21:36:02.028Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f1/12/9d49270e9c16cc4e94729c0fb2ce25e2cb7d61d4f10458c1c2fe8ab12ca1/policyengine_observability-3.0.3.tar.gz", hash = "sha256:9af61040feb9c8dc4480445dd6fee8915b2262390beec8b0a7f3bb14d52c13a0", size = 127402, upload-time = "2026-10-20T19:39:23.199Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a5/9c/b48fb64c1bebf83bd5fdeebe914188806c3513cf2de0fc95ee1d95082fc4/policyengine_observability-3.0.2-py3-none-any.whl", hash = "sha256:d84c7564922f6a3294268e89549065924edde932c31adf8b7c714558e107cb31", size = 42029, upload-time = "2026-09-30T21:36:00.841Z" }, + { url = "https://files.pythonhosted.org/packages/ff/f7/d060f939b3d80f870e084d79799dbee1674fbeae533702ba31ba3bcb8ffa/policyengine_observability-3.0.3-py3-none-any.whl", hash = "sha256:2b3276472081454095e08b8ce3c508fdeaf9a51ef9208dc3044f74076c9822ca", size = 42186, upload-time = "2026-10-20T19:39:21.775Z" }, ] [package.optional-dependencies]