From 12cc1833b4ec3c69e0ecae912246ef82486ff22b Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:42:29 +0400 Subject: [PATCH 1/4] Add telemetry helper for unhandled exceptions record_unhandled_exception records an exception that reached a catch-all error handler as an unhandled server error. Observability failures are suppressed so they cannot change the response. Co-Authored-By: Claude Opus 5.5 --- policyengine_api/observability/__init__.py | 16 +++++++++++-- policyengine_api/observability/runtime.py | 7 ++++++ tests/unit/test_observability_runtime.py | 26 ++++++++++++++++++++++ 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/policyengine_api/observability/__init__.py b/policyengine_api/observability/__init__.py index c7c1121f3..e4442cf82 100644 --- a/policyengine_api/observability/__init__.py +++ b/policyengine_api/observability/__init__.py @@ -1,5 +1,17 @@ """API observability runtime, identifiers, and registered stage plans.""" -from .runtime import _build_runtime, get_runtime, runtime, set_runtime_context +from .runtime import ( + _build_runtime, + get_runtime, + record_unhandled_exception, + runtime, + set_runtime_context, +) -__all__ = ["_build_runtime", "get_runtime", "runtime", "set_runtime_context"] +__all__ = [ + "_build_runtime", + "get_runtime", + "record_unhandled_exception", + "runtime", + "set_runtime_context", +] diff --git a/policyengine_api/observability/runtime.py b/policyengine_api/observability/runtime.py index b01840b18..edaad52cc 100644 --- a/policyengine_api/observability/runtime.py +++ b/policyengine_api/observability/runtime.py @@ -78,3 +78,10 @@ def set_runtime_context(**attributes: object) -> None: # Telemetry must never affect application behavior. with contextlib.suppress(Exception): runtime.set_context(**attributes) + + +def record_unhandled_exception(error: Exception, *, status_code: int = 500) -> None: + """Record an exception that reached a catch-all error handler.""" + # Telemetry must never affect application behavior. + with contextlib.suppress(Exception): + runtime.record_exception(error, handled=False, status_code=status_code) diff --git a/tests/unit/test_observability_runtime.py b/tests/unit/test_observability_runtime.py index 70740dfc9..74db7c72c 100644 --- a/tests/unit/test_observability_runtime.py +++ b/tests/unit/test_observability_runtime.py @@ -7,11 +7,17 @@ from policyengine_api.observability import ( _build_runtime, + record_unhandled_exception, runtime, set_runtime_context, ) +def _fail_observability(*_arguments, **_options): + message = "observability unavailable" + raise RuntimeError(message) + + def test_runtime_uses_consumer_owned_identity_and_stdout(monkeypatch): monkeypatch.setenv("OTEL_SDK_DISABLED", "true") monkeypatch.setenv("OTEL_TRACES_SAMPLER_ARG", "0.01") @@ -48,3 +54,23 @@ def test_runtime_context_failure_does_not_escape(monkeypatch): ) set_runtime_context(country_id="us") + + +def test_unhandled_exception_is_recorded_as_unhandled_server_error(monkeypatch): + recorded_calls = [] + monkeypatch.setattr( + runtime, + "record_exception", + lambda error, **options: recorded_calls.append((error, options)), + ) + error = RuntimeError("engine exploded") + + record_unhandled_exception(error) + + assert recorded_calls == [(error, {"handled": False, "status_code": 500})] + + +def test_unhandled_exception_recording_failure_does_not_escape(monkeypatch): + monkeypatch.setattr(runtime, "record_exception", _fail_observability) + + record_unhandled_exception(RuntimeError("engine exploded")) From 25947fbb544ec2932879e4778968418535d520e6 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:45:34 +0400 Subject: [PATCH 2/4] Record unhandled Flask exceptions in telemetry The catch-all Flask error handler returned a 500 without recording the exception, so telemetry only saw a failed request with no exception type or stack. It now records the exception before building the unchanged response. Co-Authored-By: Claude Opus 5.5 --- .../record-flask-exceptions.changed.md | 1 + policyengine_api/routes/error_routes.py | 2 + tests/unit/routes/test_error_routes.py | 73 +++++++++++++++++++ 3 files changed, 76 insertions(+) create mode 100644 changelog.d/record-flask-exceptions.changed.md create mode 100644 tests/unit/routes/test_error_routes.py diff --git a/changelog.d/record-flask-exceptions.changed.md b/changelog.d/record-flask-exceptions.changed.md new file mode 100644 index 000000000..48b1aa97b --- /dev/null +++ b/changelog.d/record-flask-exceptions.changed.md @@ -0,0 +1 @@ +Unhandled exceptions in Flask routes are now recorded in telemetry with their exception type and stack, so error reports can group them. diff --git a/policyengine_api/routes/error_routes.py b/policyengine_api/routes/error_routes.py index ab6005351..067d25332 100644 --- a/policyengine_api/routes/error_routes.py +++ b/policyengine_api/routes/error_routes.py @@ -5,6 +5,7 @@ HTTPException, ) +from policyengine_api.observability import record_unhandled_exception from policyengine_api.response_factory import _make_error_response error_bp = Blueprint("error", __name__) @@ -49,4 +50,5 @@ def response_http_exception(error: HTTPException) -> Response: @error_bp.app_errorhandler(Exception) def response_generic_error(error: Exception) -> Response: """Return an unhandled exception as a 500 JSON error body carrying its message.""" + record_unhandled_exception(error, status_code=500) return _make_error_response(error, 500, result=None) diff --git a/tests/unit/routes/test_error_routes.py b/tests/unit/routes/test_error_routes.py new file mode 100644 index 000000000..3afbb6f73 --- /dev/null +++ b/tests/unit/routes/test_error_routes.py @@ -0,0 +1,73 @@ +"""The Flask catch-all error handler records unhandled exceptions in telemetry.""" + +from flask import Flask + +from policyengine_api.observability import runtime +from policyengine_api.routes import error_routes + +ENGINE_FAILURE = "engine exploded" +EXISTING_500_BODY = {"status": "error", "message": ENGINE_FAILURE, "result": None} + + +def _make_client(): + app = Flask(__name__) + app.config["TESTING"] = True + app.register_blueprint(error_routes.error_bp) + + @app.route("/explode") + def explode(): + raise RuntimeError(ENGINE_FAILURE) + + return app.test_client() + + +def test_unhandled_exception_is_recorded_and_still_returns_500(monkeypatch): + recorded_calls = [] + monkeypatch.setattr( + error_routes, + "record_unhandled_exception", + lambda error, **options: recorded_calls.append((error, options)), + raising=False, + ) + + response = _make_client().get("/explode") + + assert response.status_code == 500 + assert response.get_json() == EXISTING_500_BODY + assert len(recorded_calls) == 1 + recorded_error, recorded_options = recorded_calls[0] + assert isinstance(recorded_error, RuntimeError) + assert str(recorded_error) == ENGINE_FAILURE + assert recorded_options == {"status_code": 500} + + +def test_not_found_is_not_recorded_as_unhandled_exception(monkeypatch): + recorded_calls = [] + monkeypatch.setattr( + error_routes, + "record_unhandled_exception", + lambda error, **options: recorded_calls.append((error, options)), + raising=False, + ) + + response = _make_client().get("/missing") + + assert response.status_code == 404 + assert recorded_calls == [] + + +def test_recording_failure_leaves_500_response_unchanged(monkeypatch): + attempted_errors = [] + + def fail_to_record(error, **_options): + attempted_errors.append(error) + message = "observability unavailable" + raise RuntimeError(message) + + monkeypatch.setattr(runtime, "record_exception", fail_to_record) + + response = _make_client().get("/explode") + + assert len(attempted_errors) == 1 + assert response.status_code == 500 + assert response.get_json() == EXISTING_500_BODY From 71728e2f778ed6ddb6966f349f43c0531f34e5bf Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 20 Oct 2026 22:09:54 +0400 Subject: [PATCH 3/4] Test Flask exception recording against the real observability runtime The existing tests replace runtime.record_exception with a stand-in that accepts any keyword arguments. If an installed policyengine-observability release stopped accepting a keyword that record_unhandled_exception passes, the call would raise a TypeError that the helper suppresses, recording would stop, and every test would still pass. The new test instruments a Flask app with a real runtime whose records go to an in-memory destination, raises an exception in a route, and checks that exactly one exception.recorded record is delivered with the exception type, message and stack, error.handled=false, status 500, the route, and the same request ID as the request's request.completed record. Also remove raising=False from the monkeypatch calls. error_routes now defines record_unhandled_exception, so a missing attribute should fail the test instead of being created silently. Co-Authored-By: Claude Opus 5.5 --- tests/unit/routes/test_error_routes.py | 87 +++++++++++++++++++++++++- 1 file changed, 84 insertions(+), 3 deletions(-) diff --git a/tests/unit/routes/test_error_routes.py b/tests/unit/routes/test_error_routes.py index 3afbb6f73..84eb42171 100644 --- a/tests/unit/routes/test_error_routes.py +++ b/tests/unit/routes/test_error_routes.py @@ -1,17 +1,79 @@ """The Flask catch-all error handler records unhandled exceptions in telemetry.""" +import importlib +from dataclasses import replace + +import pytest from flask import Flask +from policyengine_observability import ( + CustomLogDestination, + LoggingConfig, + OTelConfig, + configure, + instrument_flask, +) from policyengine_api.observability import runtime from policyengine_api.routes import error_routes +# policyengine_api.observability re-exports the runtime object under the name of +# its submodule, so the submodule whose global record_unhandled_exception reads +# is imported by its full name. +runtime_module = importlib.import_module("policyengine_api.observability.runtime") + ENGINE_FAILURE = "engine exploded" EXISTING_500_BODY = {"status": "error", "message": ENGINE_FAILURE, "result": None} -def _make_client(): +class _CollectedRecords: + """Keeps every structured record a runtime delivers, in delivery order.""" + + def __init__(self) -> None: + self.records: list[dict[str, object]] = [] + + def write(self, record: dict[str, object]) -> None: + self.records.append(record) + + def with_event_name(self, event_name: str) -> list[dict[str, object]]: + return [ + record for record in self.records if record.get("event.name") == event_name + ] + + +@pytest.fixture +def collected_records(): + return _CollectedRecords() + + +@pytest.fixture +def collecting_runtime(monkeypatch, collected_records): + """Point record_unhandled_exception at a real runtime that keeps its records.""" + observability_runtime = configure( + replace( + runtime.config, + logging=LoggingConfig( + destinations=( + CustomLogDestination( + name="collected-records", + writer_factory=lambda: collected_records, + delivery="inline", + ), + ), + ), + otel=OTelConfig(enabled=False), + ) + ) + monkeypatch.setattr(runtime_module, "runtime", observability_runtime) + yield observability_runtime + observability_runtime.shutdown() + + +def _make_client(observability_runtime=None): app = Flask(__name__) app.config["TESTING"] = True + if observability_runtime is not None: + # Same order as the API app: instrument Flask, then register handlers. + instrument_flask(app, observability_runtime) app.register_blueprint(error_routes.error_bp) @app.route("/explode") @@ -27,7 +89,6 @@ def test_unhandled_exception_is_recorded_and_still_returns_500(monkeypatch): error_routes, "record_unhandled_exception", lambda error, **options: recorded_calls.append((error, options)), - raising=False, ) response = _make_client().get("/explode") @@ -41,13 +102,33 @@ def test_unhandled_exception_is_recorded_and_still_returns_500(monkeypatch): assert recorded_options == {"status_code": 500} +def test_unhandled_exception_adds_one_exception_record_to_its_request( + collecting_runtime, collected_records +): + response = _make_client(collecting_runtime).get("/explode") + + assert response.status_code == 500 + assert response.get_json() == EXISTING_500_BODY + exception_records = collected_records.with_event_name("exception.recorded") + completed_records = collected_records.with_event_name("request.completed") + assert len(exception_records) == 1 + assert len(completed_records) == 1 + exception_record = exception_records[0] + assert exception_record["error.type"] == "RuntimeError" + assert exception_record["error.message"] == ENGINE_FAILURE + assert "raise RuntimeError(ENGINE_FAILURE)" in exception_record["error.stack"] + assert exception_record["error.handled"] is False + assert exception_record["http.response.status_code"] == 500 + assert exception_record["http.route"] == "/explode" + assert exception_record["request.id"] == completed_records[0]["request.id"] + + def test_not_found_is_not_recorded_as_unhandled_exception(monkeypatch): recorded_calls = [] monkeypatch.setattr( error_routes, "record_unhandled_exception", lambda error, **options: recorded_calls.append((error, options)), - raising=False, ) response = _make_client().get("/missing") From 4fb038b9b243b5ef3c561e8852a829e0e285a7c8 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 20 Oct 2026 23:43:14 +0400 Subject: [PATCH 4/4] Require policyengine-observability 3.0.3 so failed requests keep their exception In 3.0.2, Flask's teardown callback received no exception once response_generic_error had handled it, and end_request discarded the exception that record_unhandled_exception attached. The request's request.completed record then had no error fields. 3.0.3 (PolicyEngine/policyengine-observability#36) keeps the attached exception when the request has already failed. Requests that record a handled exception and still succeed keep their outcome. A new test checks that request.completed for an unhandled exception carries the exception's type, message and stack. uv.lock was regenerated with uv 0.12.1, the version CI uses, so its format revision is unchanged. The changelog fragment and gcp/observability/README.md now name 3.0.3. Co-Authored-By: Claude Opus 5.5 --- changelog.d/record-flask-exceptions.changed.md | 2 +- gcp/observability/README.md | 2 +- pyproject.toml | 2 +- tests/unit/routes/test_error_routes.py | 15 +++++++++++++++ uv.lock | 8 ++++---- 5 files changed, 22 insertions(+), 7 deletions(-) diff --git a/changelog.d/record-flask-exceptions.changed.md b/changelog.d/record-flask-exceptions.changed.md index 48b1aa97b..0a8e6bef8 100644 --- a/changelog.d/record-flask-exceptions.changed.md +++ b/changelog.d/record-flask-exceptions.changed.md @@ -1 +1 @@ -Unhandled exceptions in Flask routes are now recorded in telemetry with their exception type and stack, so error reports can group them. +Unhandled exceptions in Flask routes are now recorded in telemetry with their exception type and stack, on an exception record and on the request's completion record, so error reports can group them. This requires policyengine-observability 3.0.3. diff --git a/gcp/observability/README.md b/gcp/observability/README.md index a7d384dd3..695a57ff7 100644 --- a/gcp/observability/README.md +++ b/gcp/observability/README.md @@ -113,7 +113,7 @@ The infrastructure was applied and verified on 2026-09-22: - The alert policies have no notification channels, so they record incidents without sending email, Slack, or paging notifications. -The consumer services require `policyengine-observability` 3.0.2. Record the +The consumer services require `policyengine-observability` 3.0.3. Record the deployed consumer revisions and a representative cost and volume observation interval after the API v1 rollout. diff --git a/pyproject.toml b/pyproject.toml index a6c52ccf3..5b5a04ef1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ dependencies = [ "microdf_python>=1.0.0", "openai", "packaging>=24,<27", - "policyengine-observability[flask,google,httpx,otlp-grpc]>=3.0.2,<4", + "policyengine-observability[flask,google,httpx,otlp-grpc]>=3.0.3,<4", "policyengine_canada==0.96.3", "policyengine-ng==0.5.1", "policyengine-il==0.1.0", diff --git a/tests/unit/routes/test_error_routes.py b/tests/unit/routes/test_error_routes.py index 84eb42171..e83ab4a97 100644 --- a/tests/unit/routes/test_error_routes.py +++ b/tests/unit/routes/test_error_routes.py @@ -123,6 +123,21 @@ def test_unhandled_exception_adds_one_exception_record_to_its_request( assert exception_record["request.id"] == completed_records[0]["request.id"] +def test_unhandled_exception_is_on_its_request_completion_record( + collecting_runtime, collected_records +): + _make_client(collecting_runtime).get("/explode") + + completed_record = collected_records.with_event_name("request.completed")[0] + assert completed_record["outcome"] == "error" + assert completed_record["http.response.status_code"] == 500 + assert completed_record.get("error.type") == "RuntimeError" + assert completed_record.get("error.message") == ENGINE_FAILURE + assert "raise RuntimeError(ENGINE_FAILURE)" in completed_record.get( + "error.stack", "" + ) + + def test_not_found_is_not_recorded_as_unhandled_exception(monkeypatch): recorded_calls = [] monkeypatch.setattr( diff --git a/uv.lock b/uv.lock index 558bd32cf..0b851e2b1 100644 --- a/uv.lock +++ b/uv.lock @@ -2849,7 +2849,7 @@ requires-dist = [ { name = "policyengine-canada", specifier = "==0.96.3" }, { name = "policyengine-il", specifier = "==0.1.0" }, { name = "policyengine-ng", specifier = "==0.5.1" }, - { name = "policyengine-observability", extras = ["flask", "google", "httpx", "otlp-grpc"], specifier = ">=3.0.2,<4" }, + { name = "policyengine-observability", extras = ["flask", "google", "httpx", "otlp-grpc"], specifier = ">=3.0.3,<4" }, { name = "psycopg", extras = ["binary"], specifier = ">=3.3,<4" }, { name = "pydantic" }, { name = "pyjwt", extras = ["crypto"], specifier = ">=2.10,<3" }, @@ -2953,11 +2953,11 @@ wheels = [ [[package]] name = "policyengine-observability" -version = "3.0.2" +version = "3.0.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/74/2a/0c261c8ca693bb3a13d7fe837d3f89f7fdda245496d4b3e92eefed3c972e/policyengine_observability-3.0.2.tar.gz", hash = "sha256:554f907e43d8eeb274c1298fbe990b1626ac45f194c49995ddde091ddcdc2a64", size = 125718, upload-time = "2026-09-30T21:36:02.028Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f1/12/9d49270e9c16cc4e94729c0fb2ce25e2cb7d61d4f10458c1c2fe8ab12ca1/policyengine_observability-3.0.3.tar.gz", hash = "sha256:9af61040feb9c8dc4480445dd6fee8915b2262390beec8b0a7f3bb14d52c13a0", size = 127402, upload-time = "2026-10-20T19:39:23.199Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a5/9c/b48fb64c1bebf83bd5fdeebe914188806c3513cf2de0fc95ee1d95082fc4/policyengine_observability-3.0.2-py3-none-any.whl", hash = "sha256:d84c7564922f6a3294268e89549065924edde932c31adf8b7c714558e107cb31", size = 42029, upload-time = "2026-09-30T21:36:00.841Z" }, + { url = "https://files.pythonhosted.org/packages/ff/f7/d060f939b3d80f870e084d79799dbee1674fbeae533702ba31ba3bcb8ffa/policyengine_observability-3.0.3-py3-none-any.whl", hash = "sha256:2b3276472081454095e08b8ce3c508fdeaf9a51ef9208dc3044f74076c9822ca", size = 42186, upload-time = "2026-10-20T19:39:21.775Z" }, ] [package.optional-dependencies]